org.apache.hadoop.yarn.client.api.YarnClient Java Examples

@Test(timeout = 5000l)
public void testFailResultCodes() throws Exception {
  Configuration conf = new YarnConfiguration();
  conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
  LogCLIHelpers cliHelper = new LogCLIHelpers();
  cliHelper.setConf(conf);
  YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
  LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
  dumper.setConf(conf);
  
  // verify dumping a non-existent application's logs returns a failure code
  int exitCode = dumper.run( new String[] {
      "-applicationId", "application_0_0" } );
  assertTrue("Should return an error code", exitCode != 0);
  
  // verify dumping a non-existent container log is a failure code 
  exitCode = cliHelper.dumpAContainersLogs("application_0_0", "container_0_0",
      "nonexistentnode:1234", "nobody");
  assertTrue("Should return an error code", exitCode != 0);
}
 

@Test
public void testChangeState() throws Exception {
  YarnClient client = mock(YarnClient.class);
  YarnClusterConfiguration conf = mock(YarnClusterConfiguration.class);
  ClusterInfo clusterInfo = new ClusterInfo("foo", conf, client);
  UUID app = UUID.randomUUID();
  ApplicationId yarnAppId = mock(ApplicationId.class);

  try (InstanceManager manager = new InstanceManager(
      Collections.singletonMap("foo", clusterInfo),
      mock(InstanceStateUpdateListener.class),
      mock(ScheduledExecutorService.class),
      AthenaXExtraConfigOptions.INSTANCE_MANAGER_RESCAN_INTERVAL.defaultValue())) {
    InstanceInfo instance = new InstanceInfo("foo", yarnAppId,
        mock(InstanceMetadata.class), mock(InstanceStatus.class));
    manager.instances().put(app, instance);
    manager.changeState(app, new InstanceState().state(InstanceState.StateEnum.KILLED));
    verify(client).killApplication(eq(yarnAppId));
  }
}
 

@Override
public YarnClient get()
{
    YarnClient client = YarnClient.createYarnClient();
    if (yarnConfiguration.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
        try {
            TimelineClient.createTimelineClient();
        }
        catch (NoClassDefFoundError e) {
            logger.warn("createTimelineClient() error with {}", TimelineClient.class.getResource(TimelineClient.class.getSimpleName() + ".class"), e);
            yarnConfiguration.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
        }
    }
    client.init(yarnConfiguration);
    client.start();
    return client;
}
 

YarnJobDescriptor(
        YarnClientApplication application,
        FlinkConfiguration flinkConf,
        YarnClient yarnClient,
        YarnConfiguration yarnConfiguration,
        FlinkJobConfig appConf,
        String jobId,
        Iterable<Path> userProvidedJars)
        throws IOException
{
    super(flinkConf.flinkConfiguration(), yarnConfiguration, flinkConf.getConfigurationDirectory(), yarnClient, false);
    this.application = application;
    this.jobName = jobId;
    this.flinkConf = flinkConf;
    this.yarnClient = yarnClient;
    this.appConf = appConf;
    this.userProvidedJars = userProvidedJars;

    FileSystem fileSystem = FileSystem.get(yarnClient.getConfig());
    this.uploadingDir = new Path(new Path(fileSystem.getHomeDirectory(), ".sylph"), application.getApplicationSubmissionContext().getApplicationId().toString());
}
 

private int verifyApplicationState(ApplicationId appId) throws IOException,
    YarnException {
  YarnClient yarnClient = createYarnClient();

  try {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    switch (appReport.getYarnApplicationState()) {
    case NEW:
    case NEW_SAVING:
    case SUBMITTED:
      return -1;
    case ACCEPTED:
    case RUNNING:
    case FAILED:
    case FINISHED:
    case KILLED:
    default:
      break;

    }
  } finally {
    yarnClient.close();
  }
  return 0;
}
 

@Test
public void testGetEmptyClusterNodeLabels() throws Exception {
  YarnClient client = mock(YarnClient.class);
  when(client.getClusterNodeLabels()).thenReturn(new HashSet<String>());
  ClusterCLI cli = new ClusterCLI();
  cli.setClient(client);
  cli.setSysOutPrintStream(sysOut);
  cli.setSysErrPrintStream(sysErr);

  int rc =
      cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD });
  assertEquals(0, rc);

  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  PrintWriter pw = new PrintWriter(baos);
  pw.print("Node Labels: ");
  pw.close();
  verify(sysOut).println(baos.toString("UTF-8"));
}
 

@Test
public void testKillApplication() throws Exception {
  MockRM rm = new MockRM();
  rm.start();
  RMApp app = rm.submitApp(2000);

  Configuration conf = new Configuration();
  @SuppressWarnings("resource")
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  client.killApplication(app.getApplicationId());
  verify(((MockYarnClient) client).getRMClient(), times(2))
    .forceKillApplication(any(KillApplicationRequest.class));
}
 

@Test
public void testGetClusterNodeLabelsWithLocalAccess() throws Exception {
  YarnClient client = mock(YarnClient.class);
  when(client.getClusterNodeLabels()).thenReturn(
      ImmutableSet.of("remote1", "remote2"));
  ClusterCLI cli = new ClusterCLI();
  cli.setClient(client);
  cli.setSysOutPrintStream(sysOut);
  cli.setSysErrPrintStream(sysErr);
  ClusterCLI.localNodeLabelsManager = mock(CommonNodeLabelsManager.class);
  when(ClusterCLI.localNodeLabelsManager.getClusterNodeLabels())
      .thenReturn(ImmutableSet.of("local1", "local2"));

  int rc =
      cli.run(new String[] { ClusterCLI.CMD,
          "-" + ClusterCLI.LIST_LABELS_CMD,
          "-" + ClusterCLI.DIRECTLY_ACCESS_NODE_LABEL_STORE });
  assertEquals(0, rc);

  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  PrintWriter pw = new PrintWriter(baos);
  // it should return local* instead of remote*
  pw.print("Node Labels: local1,local2");
  pw.close();
  verify(sysOut).println(baos.toString("UTF-8"));
}
 

/**
 * Adds RM delegation token to the given {@link ContainerLaunchContext} so that the AM can authenticate itself
 * with RM using the delegation token.
 */
protected void addRMToken(ContainerLaunchContext context, YarnClient yarnClient, ApplicationId appId) {
  if (!UserGroupInformation.isSecurityEnabled()) {
    return;
  }

  try {
    Credentials credentials = YarnUtils.decodeCredentials(context.getTokens());

    Configuration config = yarnClient.getConfig();
    Token<TokenIdentifier> token = ConverterUtils.convertFromYarn(
      yarnClient.getRMDelegationToken(new Text(YarnUtils.getYarnTokenRenewer(config))),
      YarnUtils.getRMAddress(config));

    LOG.debug("Added RM delegation token {} for application {}", token, appId);
    credentials.addToken(token.getService(), token);

    context.setTokens(YarnUtils.encodeCredentials(credentials));
  } catch (YarnException | IOException e) {
    throw new RuntimeException("Failed to acquire RM delegation token", e);
  }
}
 

public AbstractYarnClusterDescriptor(
		Configuration flinkConfiguration,
		YarnConfiguration yarnConfiguration,
		String configurationDirectory,
		YarnClient yarnClient,
		boolean sharedYarnClient) {

	this.yarnConfiguration = Preconditions.checkNotNull(yarnConfiguration);

	// for unit tests only
	if (System.getenv("IN_TESTS") != null) {
		try {
			yarnConfiguration.addResource(new File(System.getenv("YARN_CONF_DIR"), "yarn-site.xml").toURI().toURL());
		} catch (Throwable t) {
			throw new RuntimeException("Error", t);
		}
	}

	this.yarnClient = Preconditions.checkNotNull(yarnClient);
	this.sharedYarnClient = sharedYarnClient;

	this.flinkConfiguration = Preconditions.checkNotNull(flinkConfiguration);
	userJarInclusion = getUserJarInclusionMode(flinkConfiguration);

	this.configurationDirectory = Preconditions.checkNotNull(configurationDirectory);
}
 

@Test (timeout = 10000)
public void testGetLabelsToNodes() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  // Get labels to nodes mapping
  Map<String, Set<NodeId>> expectedLabelsToNodes =
      ((MockYarnClient)client).getLabelsToNodesMap();
  Map<String, Set<NodeId>> labelsToNodes = client.getLabelsToNodes();
  Assert.assertEquals(labelsToNodes, expectedLabelsToNodes);
  Assert.assertEquals(labelsToNodes.size(), 3);

  // Get labels to nodes for selected labels
  Set<String> setLabels = new HashSet<String>(Arrays.asList("x", "z"));
  expectedLabelsToNodes =
      ((MockYarnClient)client).getLabelsToNodesMap(setLabels);
  labelsToNodes = client.getLabelsToNodes(setLabels);
  Assert.assertEquals(labelsToNodes, expectedLabelsToNodes);
  Assert.assertEquals(labelsToNodes.size(), 2);

  client.stop();
  client.close();
}
 

@Test(timeout = 10000)
public void testGetApplicationAttempt() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports = ((MockYarnClient) client)
      .getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ApplicationAttemptReport report = client
      .getApplicationAttemptReport(appAttemptId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getApplicationAttemptId().toString(),
      expectedReports.get(0).getCurrentApplicationAttemptId().toString());
  client.stop();
}
 

@Test(timeout = 5000l)
public void testFailResultCodes() throws Exception {
  Configuration conf = new YarnConfiguration();
  conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
  LogCLIHelpers cliHelper = new LogCLIHelpers();
  cliHelper.setConf(conf);
  YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
  LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
  dumper.setConf(conf);
  
  // verify dumping a non-existent application's logs returns a failure code
  int exitCode = dumper.run( new String[] {
      "-applicationId", "application_0_0" } );
  assertTrue("Should return an error code", exitCode != 0);
  
  // verify dumping a non-existent container log is a failure code 
  exitCode = cliHelper.dumpAContainersLogs("application_0_0", "container_0_0",
      "nonexistentnode:1234", "nobody");
  assertTrue("Should return an error code", exitCode != 0);
}
 

public static List<ApplicationReport> cleanAppDirectories(YarnClient clientRMService, Configuration conf, FileSystem fs, long finishedBefore)
    throws IOException, YarnException
{
  List<ApplicationReport> result = new ArrayList<>();
  List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED),
      EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.FINISHED, YarnApplicationState.KILLED));
  Path appsBasePath = new Path(StramClientUtils.getApexDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS);
  for (ApplicationReport ar : applications) {
    long finishTime = ar.getFinishTime();
    if (finishTime < finishedBefore) {
      try {
        Path appPath = new Path(appsBasePath, ar.getApplicationId().toString());
        if (fs.isDirectory(appPath)) {
          LOG.debug("Deleting finished application data for {}", ar.getApplicationId());
          fs.delete(appPath, true);
          result.add(ar);
        }
      } catch (Exception ex) {
        LOG.warn("Cannot delete application data for {}", ar.getApplicationId(), ex);
        continue;
      }
    }
  }
  return result;
}
 

@Test
public void testCreateAthenaXCluster() throws Exception {
  ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
  Configuration flinkConf = new Configuration();
  flinkConf.setString(JobManagerOptions.ADDRESS, "localhost");

  try (MiniAthenaXCluster cluster = new MiniAthenaXCluster(JobDeployerITest.class.getSimpleName())) {
    cluster.start();
    YarnConfiguration conf = cluster.getYarnConfiguration();
    YarnClusterConfiguration clusterConf = cluster.getYarnClusterConf();

    final ApplicationId appId;
    try (YarnClient client = YarnClient.createYarnClient()) {
      client.init(conf);
      client.start();

      JobDeployer deployer = new JobDeployer(clusterConf, client, executor, flinkConf);
      appId = deployer.createApplication();
      InstanceMetadata md = new InstanceMetadata(UUID.randomUUID(), UUID.randomUUID());
      JobDefinitionResource resource = new JobDefinitionResource()
          .queue(null).vCores(1L).executionSlots(1L).memory(2048L);
      JobConf jobConf = new JobConf(appId, "test", Collections.emptyList(), resource, md);
      deployer.start(JobITestUtil.trivialJobGraph(), jobConf);

      YarnApplicationState state = MiniAthenaXCluster.pollFinishedApplicationState(client, appId);
      assertEquals(FINISHED, state);
    }
  }
}
 

ClusterInfo(String name, YarnClusterConfiguration conf) {
  this.name = name;
  this.client = YarnClient.createYarnClient();
  client.init(conf.conf());
  client.start();
  this.conf = conf;
}
 

@Test(timeout = 5000l)
public void testHelpMessage() throws Exception {
  Configuration conf = new YarnConfiguration();
  YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
  LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
  dumper.setConf(conf);

  int exitCode = dumper.run(new String[]{});
  assertTrue(exitCode == -1);
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  PrintWriter pw = new PrintWriter(baos);
  pw.println("Retrieve logs for completed YARN applications.");
  pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]");
  pw.println();
  pw.println("general options are:");
  pw.println(" -appOwner <Application Owner>   AppOwner (assumed to be current user if");
  pw.println("                                 not specified)");
  pw.println(" -containerId <Container ID>     ContainerId (must be specified if node");
  pw.println("                                 address is specified)");
  pw.println(" -help                           Displays help for all commands.");
  pw.println(" -nodeAddress <Node Address>     NodeAddress in the format nodename:port");
  pw.println("                                 (must be specified if container id is");
  pw.println("                                 specified)");
  pw.close();
  String appReportStr = baos.toString("UTF-8");
  Assert.assertEquals(appReportStr, sysOutStream.toString());
}
 

private YarnClusterDescriptor getClusterDescriptor(Configuration configuration) {
	final YarnClient yarnClient = YarnClient.createYarnClient();
	final YarnConfiguration yarnConfiguration = new YarnConfiguration();

	yarnClient.init(yarnConfiguration);
	yarnClient.start();

	return new YarnClusterDescriptor(
			configuration,
			yarnConfiguration,
			yarnClient,
			YarnClientYarnClusterInformationRetriever.create(yarnClient),
			false);
}
 

@Override
public void terminateJob(Properties jobProps) throws IOException {
  String appIdStr = Utils.checkNotNull(jobProps.getProperty("appId"), null);
  if (appIdStr == null) {
    throw new IOException("appId not present in jobProps");
  }
  YarnClient yarnClient = getYarnClient(new EMRJobConfig(jobProps).getClusterId(), emrClusterConfig);
  ApplicationId appId = ApplicationId.fromString(appIdStr);
  try {
    yarnClient.killApplication(appId);
  } catch (YarnException ex) {
    throw new IOException("Failed to terminate yarn job " + ex);
  }
}
 

private YarnClient createMockYarnClientUnknownApp() throws YarnException,
    IOException {
  YarnClient mockClient = mock(YarnClient.class);
  doThrow(new YarnException("Unknown AppId")).when(mockClient)
      .getApplicationReport(any(ApplicationId.class));
  return mockClient;
}
 

@BeforeClass
public static void setup() throws Exception {
  // start minicluster
  conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
    rolling_interval_sec);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
  conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();

  // start rm client
  yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
  
  priority = Priority.newInstance(1);
  priority2 = Priority.newInstance(2);
  capability = Resource.newInstance(1024, 1);

  node = nodeReports.get(0).getNodeId().getHost();
  rack = nodeReports.get(0).getRackName();
  nodes = new String[]{ node };
  racks = new String[]{ rack };
}
 

@Override
public void cancel() {
  YarnClient yarnClient = createYarnClient();
  try {
    yarnClient.killApplication(appId);
  } catch (YarnException | IOException e) {
    throw new RuntimeException("Failed to kill application " + appId, e);
  } finally {
    yarnClient.stop();
  }
}
 

@Test(timeout = 10000)
public void testGetContainerReport() throws YarnException, IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
      true);
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports = ((MockYarnClient) client)
      .getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  ContainerReport report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 1)).toString());
  containerId = ContainerId.newContainerId(appAttemptId, 3);
  report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 3)).toString());
  client.stop();
}
 

@Test(timeout = 5000l)
public void testUnknownApplicationId() throws Exception {
  Configuration conf = new YarnConfiguration();
  YarnClient mockYarnClient = createMockYarnClientUnknownApp();
  LogsCLI cli = new LogsCLIForTest(mockYarnClient);
  cli.setConf(conf);

  int exitCode = cli.run(new String[] { "-applicationId",
      ApplicationId.newInstance(1, 1).toString() });

  // Error since no logs present for the app.
  assertTrue(exitCode != 0);
  assertTrue(sysErrStream.toString().startsWith(
      "Unable to get ApplicationState"));
}
 

public NonDeployingYarnClusterDescriptor(
		Configuration flinkConfiguration,
		YarnConfiguration yarnConfiguration,
		String configurationDirectory,
		YarnClient yarnClient,
		ClusterClient<ApplicationId> clusterClient) {
	super(flinkConfiguration, yarnConfiguration, configurationDirectory, yarnClient, true);

	//noinspection unchecked
	this.clusterClient = Preconditions.checkNotNull(clusterClient);
}
 

/**
 * Kills YARN application and stops YARN client.
 *
 * <p>Use this method to kill the App before it has been properly deployed
 */
private void failSessionDuringDeployment(YarnClient yarnClient, YarnClientApplication yarnApplication) {
	LOG.info("Killing YARN application");

	try {
		yarnClient.killApplication(yarnApplication.getNewApplicationResponse().getApplicationId());
	} catch (Exception e) {
		// we only log a debug message here because the "killApplication" call is a best-effort
		// call (we don't know if the application has been deployed when the error occured).
		LOG.debug("Error while killing YARN application", e);
	}
}
 

@Override
public List<NodeReport> getNodeReports() throws Exception {
  YarnClient yarnClient = createYarnClient();
  try {
    return yarnClient.getNodeReports();
  } finally {
    yarnClient.stop();
  }
}
 

NonDeployingDetachedYarnClusterDescriptor(
	Configuration flinkConfiguration,
	YarnConfiguration yarnConfiguration, String configurationDirectory,
	YarnClient yarnClient,
	ClusterClient<ApplicationId> clusterClient) {
	super(flinkConfiguration, yarnConfiguration, configurationDirectory, yarnClient,
		clusterClient);
}
 

@Before
public void setupYarnClient() {
	if (yarnClient == null) {
		yarnClient = YarnClient.createYarnClient();
		yarnClient.init(getYarnConfiguration());
		yarnClient.start();
	}

	flinkConfiguration = new org.apache.flink.configuration.Configuration(globalConfiguration);
}
 

Client(Configuration conf, YarnClient yarnClient) {
  super("Client");
  this.conf = conf;
  this.yarnClient = yarnClient;
  yarnClient.init(conf);
  yarnClient.start();
  LOG.info("Starting YarnClient...");
}