Java Code Examples for org.apache.hadoop.yarn.client.api.YarnClient#getApplicationReport()

private void waitTillAccepted(YarnClient rmClient, ApplicationId appId)
  throws Exception {
  try {
    long start = System.currentTimeMillis();
    ApplicationReport report = rmClient.getApplicationReport(appId);
    while (YarnApplicationState.ACCEPTED != report.getYarnApplicationState()) {
      if (System.currentTimeMillis() - start > 20 * 1000) {
        throw new Exception("App '" + appId + 
          "' time out, failed to reach ACCEPTED state");
      }
      Thread.sleep(200);
      report = rmClient.getApplicationReport(appId);
    }
  } catch (Exception ex) {
    throw new Exception(ex);
  }
}
 

private int verifyApplicationState(ApplicationId appId) throws IOException,
    YarnException {
  YarnClient yarnClient = createYarnClient();

  try {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    switch (appReport.getYarnApplicationState()) {
    case NEW:
    case NEW_SAVING:
    case SUBMITTED:
      return -1;
    case ACCEPTED:
    case RUNNING:
    case FAILED:
    case FINISHED:
    case KILLED:
    default:
      break;

    }
  } finally {
    yarnClient.close();
  }
  return 0;
}
 

private int verifyApplicationState(ApplicationId appId) throws IOException,
    YarnException {
  YarnClient yarnClient = createYarnClient();

  try {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    switch (appReport.getYarnApplicationState()) {
    case NEW:
    case NEW_SAVING:
    case SUBMITTED:
      return -1;
    case ACCEPTED:
    case RUNNING:
    case FAILED:
    case FINISHED:
    case KILLED:
    default:
      break;

    }
  } finally {
    yarnClient.close();
  }
  return 0;
}
 

private void waitTillAccepted(YarnClient rmClient, ApplicationId appId)
  throws Exception {
  try {
    long start = System.currentTimeMillis();
    ApplicationReport report = rmClient.getApplicationReport(appId);
    while (YarnApplicationState.ACCEPTED != report.getYarnApplicationState()) {
      if (System.currentTimeMillis() - start > 20 * 1000) {
        throw new Exception("App '" + appId + 
          "' time out, failed to reach ACCEPTED state");
      }
      Thread.sleep(200);
      report = rmClient.getApplicationReport(appId);
    }
  } catch (Exception ex) {
    throw new Exception(ex);
  }
}
 

private void tryRetrieveAndLogApplicationReport(YarnClient yarnClient, ApplicationId yarnApplicationId) {
	ApplicationReport applicationReport;

	try {
		applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
	} catch (YarnException | IOException e) {
		LOG.info("Could not log the final application report.", e);
		applicationReport = null;
	}

	if (applicationReport != null) {
		logApplicationReport(applicationReport);
	}
}
 

static DAGClientAMProtocolBlockingPB getSessionAMProxy(YarnClient yarnClient,
    Configuration conf,
    ApplicationId applicationId) throws TezException, IOException {
  ApplicationReport appReport;
  try {
    appReport = yarnClient.getApplicationReport(
        applicationId);

    if(appReport == null) {
      throw new TezUncheckedException("Could not retrieve application report"
          + " from YARN, applicationId=" + applicationId);
    }
    YarnApplicationState appState = appReport.getYarnApplicationState();
    if(appState != YarnApplicationState.RUNNING) {
      if (appState == YarnApplicationState.FINISHED
          || appState == YarnApplicationState.KILLED
          || appState == YarnApplicationState.FAILED) {
        throw new SessionNotRunning("Application not running"
            + ", applicationId=" + applicationId
            + ", yarnApplicationState=" + appReport.getYarnApplicationState()
            + ", finalApplicationStatus="
            + appReport.getFinalApplicationStatus()
            + ", trackingUrl=" + appReport.getTrackingUrl());
      }
      return null;
    }
  } catch (YarnException e) {
    throw new TezException(e);
  }
  return getAMProxy(conf, appReport.getHost(),
      appReport.getRpcPort(), appReport.getClientToAMToken());
}
 

private void tryRetrieveAndLogApplicationReport(YarnClient yarnClient, ApplicationId yarnApplicationId) {
	ApplicationReport applicationReport;

	try {
		applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
	} catch (YarnException | IOException e) {
		LOG.info("Could not log the final application report.", e);
		applicationReport = null;
	}

	if (applicationReport != null) {
		logApplicationReport(applicationReport);
	}
}
 

@Override
public YarnApplicationReport getReport() {
  YarnClient yarnClient = createYarnClient();
  try {
    return new Hadoop21YarnApplicationReport(yarnClient.getApplicationReport(appId));
  } catch (YarnException | IOException e) {
    throw new RuntimeException("Failed to get application report for " + appId, e);
  } finally {
    yarnClient.stop();
  }
}
 

static ClusterSpec getClusterSpec(YarnClient client, ApplicationId appId) throws Exception {
  ClusterSpec clusterSpec = ClusterSpec.empty();
  ApplicationReport report = client.getApplicationReport(appId);
  YarnApplicationState state = report.getYarnApplicationState();
  if (state.equals(YarnApplicationState.RUNNING)) {
    String hostname = report.getHost();
    int port = report.getRpcPort();
    TFApplicationRpc rpc = TFApplicationRpcClient.getInstance(hostname, port);
    String spec = rpc.getClusterSpec();
    if (spec != null) {
      clusterSpec = ClusterSpec.fromJsonString(spec);
    }
  }
  return clusterSpec;
}
 

public static YarnApplicationState pollFinishedApplicationState(YarnClient client, ApplicationId appId)
    throws IOException, YarnException, InterruptedException {
  EnumSet<YarnApplicationState> finishedState = EnumSet.of(FINISHED, KILLED, FAILED);

  while (true) {
    ApplicationReport report = client.getApplicationReport(appId);
    YarnApplicationState state = report.getYarnApplicationState();
    if (finishedState.contains(state)) {
      return state;
    } else {
      Thread.sleep(250);
    }
  }
}
 

private void tryRetrieveAndLogApplicationReport(YarnClient yarnClient, ApplicationId yarnApplicationId) {
	ApplicationReport applicationReport;

	try {
		applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
	} catch (YarnException | IOException e) {
		LOG.info("Could not log the final application report.", e);
		applicationReport = null;
	}

	if (applicationReport != null) {
		logApplicationReport(applicationReport);
	}
}
 

private static ApplicationReport getApplicationReport(ApplicationId appId, YarnClient yarnClient)
    throws YarnException, IOException {
  return yarnClient.getApplicationReport(appId);
}
 

public void run(String[] args) throws Exception {
    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    final Path jarPath = new Path(args[2]);
    final String resourceManagerAddress = args[3];
    final String resourceManagerHostname = args[4];
    final String resourceManagerSchedulerAddress = args[5];
    final String resourceManagerResourceTrackerAddress = args[6];

    // Create yarnClient
    YarnConfiguration conf = new YarnConfiguration();
    conf.set("yarn.resourcemanager.address", resourceManagerAddress);
    conf.set("yarn.resourcemanager.hostname", resourceManagerHostname);
    conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress);
    conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer =
            Records.newRecord(ContainerLaunchContext.class);
    amContainer.setCommands(
            Collections.singletonList(
                    "$JAVA_HOME/bin/java" +
                            " -Xmx256M" +
                            " com.hortonworks.simpleyarnapp.ApplicationMaster" +
                            " " + command +
                            " " + String.valueOf(n) +
                            " " + resourceManagerAddress +
                            " " + resourceManagerHostname +
                            " " + resourceManagerSchedulerAddress +
                            " " + resourceManagerResourceTrackerAddress +
                            " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
                            " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
            )
    );

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = Records.newRecord(LocalResource.class);
    setupAppMasterJar(jarPath, appMasterJar);
    amContainer.setLocalResources(
            Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext =
            app.getApplicationSubmissionContext();
    appContext.setApplicationName("simple-yarn-app"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();
    System.out.println("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED &&
            appState != YarnApplicationState.KILLED &&
            appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println(
            "Application " + appId + " finished with" +
                    " state " + appState +
                    " at " + appReport.getFinishTime());

}
 

@Override
public Properties getJobStatus(Properties jobProps) throws IOException {
  EMRJobConfig emrJobConfig = new EMRJobConfig(jobProps);
  Utils.checkNotNull(emrJobConfig.getClusterId(), "EMR Cluster Id");
  String state;
  String message = null;
  DescribeStepResult res = getEmrClient(emrClusterConfig).describeStep(new DescribeStepRequest().withClusterId(
      emrJobConfig.getClusterId()).withStepId(emrJobConfig.getStepId()));
  StepStatus status = res.getStep().getStatus();
  ApplicationId appId = null;
  LOG.debug(Utils.format("Status of step: {} is {}", emrJobConfig.getStepId(), status.getState()));
  if ("PENDING".equals(status.getState())) {
    state = status.getState();
    if (status.getStateChangeReason() != null) {
      message = status.getStateChangeReason().getMessage();
    }
  } else if (!"COMPLETED".equals(status.getState()) && !"RUNNING".equals(status.getState())) {
    state = status.getState();
    if (status.getFailureDetails() != null) {
      message = status.getFailureDetails().getReason();
    }
  } else {
    YarnClient yarnClient = getYarnClient(emrJobConfig.getClusterId(), emrClusterConfig);
    ApplicationReport report = null;
    try {
      for (ApplicationReport applicationReport : yarnClient.getApplications()) {
        if (applicationReport.getName().contains(emrJobConfig.getUniquePrefix())) {
          appId = applicationReport.getApplicationId();
          break;
        }
      }
      if (appId != null) {
        report = yarnClient.getApplicationReport(appId);
      }
    } catch (YarnException ex) {
      throw new IOException("Failed to fetch yarn app report " + ex);
    }
    if (report != null) {
      YarnApplicationState yarnState = report.getYarnApplicationState();
      FinalApplicationStatus finalApplicationStatus = report.getFinalApplicationStatus();
      LOG.info(Utils.format("Application state for app id: {} is {} ", appId, yarnState));
      state = yarnState.name();
      if (YarnApplicationState.FINISHED == yarnState) {
        // override with final application state if yarnState is FINISHED
        state = finalApplicationStatus.name();
      }
      message = report.getDiagnostics();
    } else {
      state = "STARTING"; // a situation where step was in RUNNING but yarn application not yet created.
      message = "Yarn application not yet created";
    }
  }
  EmrState emrJobState = new EmrState();
  emrJobState.setState(state);
  emrJobState.setMessage(message);
  emrJobState.setAppId(appId != null ? appId.toString() : null);
  return emrJobState.toProperties();
}
 

@Test(timeout = 60000)
public void testAMClientHeartbeatTimeout() throws Exception {
  Path stagingDirPath = new Path("/tmp/timeout-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  YarnClient yarnClient = YarnClient.createYarnClient();

  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    tezConf.setInt(TezConfiguration.TEZ_AM_CLIENT_HEARTBEAT_TIMEOUT_SECS, 5);
    TezClient tezClient = TezClient.create("testAMClientHeartbeatTimeout", tezConf, true);
    tezClient.start();
    tezClient.cancelAMKeepAlive(true);

    ApplicationId appId = tezClient.getAppMasterApplicationId();

    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;

    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);

    ApplicationReport report;
    while (true) {
      report = yarnClient.getApplicationReport(appId);
      if (report.getYarnApplicationState() == YarnApplicationState.FINISHED
          || report.getYarnApplicationState() == YarnApplicationState.FAILED
          || report.getYarnApplicationState() == YarnApplicationState.KILLED) {
        break;
      }
      Thread.sleep(1000);
    }
    // Add a sleep because YARN is not consistent in terms of reporting uptodate diagnostics
    Thread.sleep(2000);
    report = yarnClient.getApplicationReport(appId);
    LOG.info("App Report for appId=" + appId
        + ", report=" + report);
    Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(),
        report.getDiagnostics().contains("Client-to-AM Heartbeat timeout interval expired"));

  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}
 

@Test(timeout = 60000)
public void testSessionTimeout() throws Exception {
  Path stagingDirPath = new Path("/tmp/sessiontimeout-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  YarnClient yarnClient = YarnClient.createYarnClient();

  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    tezConf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, 5);
    TezClient tezClient = TezClient.create("testSessionTimeout", tezConf, true);
    tezClient.start();

    ApplicationId appId = tezClient.getAppMasterApplicationId();

    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;

    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);

    ApplicationReport report;
    while (true) {
      report = yarnClient.getApplicationReport(appId);
      if (report.getYarnApplicationState() == YarnApplicationState.FINISHED
          || report.getYarnApplicationState() == YarnApplicationState.FAILED
          || report.getYarnApplicationState() == YarnApplicationState.KILLED) {
        break;
      }
      Thread.sleep(1000);
    }
    // Add a sleep because YARN is not consistent in terms of reporting uptodate diagnostics
    Thread.sleep(2000);
    report = yarnClient.getApplicationReport(appId);
    LOG.info("App Report for appId=" + appId
        + ", report=" + report);
    Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(),
        report.getDiagnostics().contains("Session timed out"));

  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}
 

/**
 * verify the diagnostics in {@link DAGStatus} is correct in non-session mode,
 * and also verify that diagnostics from {@link DAGStatus} should match that
 * from {@link ApplicationReport}
 * 
 * @throws Exception
 */
@Test(timeout = 120000)
public void testExceptionPropagationNonSession() throws Exception {
  try {
    startMiniTezCluster();
    startNonSessionClient();

    ExceptionLocation exLocation = ExceptionLocation.EM_GetNumSourceTaskPhysicalOutputs;
    LOG.info("NonSession mode, Test for Exception from:" + exLocation.name());
    DAG dag = createDAG(exLocation);
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGStatus dagStatus = dagClient.waitForCompletion();
    String diagnostics = StringUtils.join(dagStatus.getDiagnostics(), ",");
    LOG.info("Diagnostics:" + diagnostics);
    assertTrue(diagnostics.contains(exLocation.name()));

    // wait for app complete (unregisterApplicationMaster is done)
    ApplicationId appId = tezClient.getAppMasterApplicationId();
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(tezConf);
    yarnClient.start();
    Set<YarnApplicationState> FINAL_APPLICATION_STATES =
        EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FAILED,
            YarnApplicationState.FINISHED);
    ApplicationReport appReport = null;
    while (true) {
      appReport = yarnClient.getApplicationReport(appId);
      Thread.sleep(1000);
      LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus());
      LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics());
      if (FINAL_APPLICATION_STATES.contains(appReport
          .getYarnApplicationState())) {
        break;
      }
    }
    // wait for 1 second and call getApplicationReport again to ensure get the
    // diagnostics
    // TODO remove it after YARN-2560
    Thread.sleep(1000);
    appReport = yarnClient.getApplicationReport(appId);

    LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus());
    LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics());
    assertTrue(appReport.getDiagnostics().contains(exLocation.name()));
    // use "\n" as separator, because we also use it in Tez internally when
    // assembling the application diagnostics.
    assertEquals(StringUtils.join(dagStatus.getDiagnostics(), "\n").trim(),
        appReport.getDiagnostics().trim());
  } finally {
    stopNonSessionClient();
    stopTezMiniCluster();
  }
}
 

@Test
public void testAmShutdown() throws IOException, YarnException {

  Assume.assumeTrue(
      "This test requires a YARN Resource Manager to connect to",
      Boolean.parseBoolean(System.getenv("REEF_TEST_YARN")));

  final YarnConfiguration yarnConfig = new YarnConfiguration();

  // Start YARN client and register the application

  final YarnClient yarnClient = YarnClient.createYarnClient();
  yarnClient.init(yarnConfig);
  yarnClient.start();

  final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class);
  containerContext.setCommands(Collections.<String>emptyList());
  containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap());
  containerContext.setEnvironment(Collections.<String, String>emptyMap());
  containerContext.setTokens(getTokens());

  final ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext();
  appContext.setApplicationName("REEF_Unmanaged_AM_Test");
  appContext.setAMContainerSpec(containerContext);
  appContext.setUnmanagedAM(true);
  appContext.setQueue("default");

  final ApplicationId applicationId = appContext.getApplicationId();
  LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId);

  yarnClient.submitApplication(appContext);

  LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId);

  addToken(yarnClient.getAMRMToken(applicationId));

  // Start the AM

  final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this);
  rmClient.init(yarnConfig);
  rmClient.start();

  final NMClientAsync nmClient = new NMClientAsyncImpl(this);
  nmClient.init(yarnConfig);
  nmClient.start();

  final RegisterApplicationMasterResponse registration =
      rmClient.registerApplicationMaster(NetUtils.getHostname(), -1, null);

  LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration);

  rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null);

  LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState());

  // Shutdown the AM

  rmClient.stop();
  nmClient.stop();

  // Get the final application report

  final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);
  final YarnApplicationState appState = appReport.getYarnApplicationState();
  final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus();

  LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] {
      applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus});

  Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState);
  Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus);

  // Shutdown YARN client

  yarnClient.stop();
}