Java Code Examples for org.apache.hadoop.yarn.client.api.YarnClient#stop()

The following examples show how to use org.apache.hadoop.yarn.client.api.YarnClient#stop() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestYarnClient.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 10000)
public void testGetApplicationAttempts() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  List<ApplicationAttemptReport> reports = client
      .getApplicationAttempts(applicationId);
  Assert.assertNotNull(reports);
  Assert.assertEquals(reports.get(0).getApplicationAttemptId(),
      ApplicationAttemptId.newInstance(applicationId, 1));
  Assert.assertEquals(reports.get(1).getApplicationAttemptId(),
      ApplicationAttemptId.newInstance(applicationId, 2));
  client.stop();
}
Example 2
Source File: YarnBinding.java    From mr4c with Apache License 2.0 6 votes vote down vote up 
public void addClusterLimits(JobConf jobConf, ResourceConfig resConf) throws IOException {
	if ( resConf.isEmpty() ) {
		return;
	}
	Configuration conf = new Configuration(jobConf);
	YarnClient client = YarnClient.createYarnClient();
	client.init(conf);
	client.start();
	try {
		for (ResourceLimit limit : computeResourceLimits(client.getNodeReports()) ) {
			resConf.addLimit(limit);
		}
	} catch ( YarnException ye ) {
		throw new IOException(ye);
	} finally {
		client.stop();
	}
}
Example 3
Source File: TestRMFailover.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);
    client.start();
    try {
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e);
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}
Example 4
Source File: TestYarnClient.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 10000)
public void testGetApplicationAttempt() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports = ((MockYarnClient) client)
      .getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ApplicationAttemptReport report = client
      .getApplicationAttemptReport(appAttemptId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getApplicationAttemptId().toString(),
      expectedReports.get(0).getCurrentApplicationAttemptId().toString());
  client.stop();
}
Example 5
Source File: ProtocolHATestBase.java    From big-c with Apache License 2.0 6 votes vote down vote up 
protected void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = createAndStartYarnClient(conf);
    try {
      Thread.sleep(100);
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e.getMessage());
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}
Example 6
Source File: TestYarnClient.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 10000)
public void testGetApplicationAttempts() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  List<ApplicationAttemptReport> reports = client
      .getApplicationAttempts(applicationId);
  Assert.assertNotNull(reports);
  Assert.assertEquals(reports.get(0).getApplicationAttemptId(),
      ApplicationAttemptId.newInstance(applicationId, 1));
  Assert.assertEquals(reports.get(1).getApplicationAttemptId(),
      ApplicationAttemptId.newInstance(applicationId, 2));
  client.stop();
}
Example 7
Source File: TestYarnClient.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test(timeout = 10000)
public void testGetApplicationAttempt() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports = ((MockYarnClient) client)
      .getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ApplicationAttemptReport report = client
      .getApplicationAttemptReport(appAttemptId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getApplicationAttemptId().toString(),
      expectedReports.get(0).getCurrentApplicationAttemptId().toString());
  client.stop();
}
Example 8
Source File: TestRMFailover.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);
    client.start();
    try {
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e);
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}
Example 9
Source File: TestYarnClient.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 10000)
public void testGetContainerReport() throws YarnException, IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
      true);
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports = ((MockYarnClient) client)
      .getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  ContainerReport report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 1)).toString());
  containerId = ContainerId.newContainerId(appAttemptId, 3);
  report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 3)).toString());
  client.stop();
}
Example 10
Source File: TestYarnClient.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testClientStop() {
  Configuration conf = new Configuration();
  ResourceManager rm = new ResourceManager();
  rm.init(conf);
  rm.start();

  YarnClient client = YarnClient.createYarnClient();
  client.init(conf);
  client.start();
  client.stop();
  rm.stop();
}
Example 11
Source File: Hadoop21YarnAppClient.java    From twill with Apache License 2.0 5 votes vote down vote up 
@Override
public void cancel() {
  YarnClient yarnClient = createYarnClient();
  try {
    yarnClient.killApplication(appId);
  } catch (YarnException | IOException e) {
    throw new RuntimeException("Failed to kill application " + appId, e);
  } finally {
    yarnClient.stop();
  }
}
Example 12
Source File: Hadoop21YarnAppClient.java    From twill with Apache License 2.0 5 votes vote down vote up 
@Override
public YarnApplicationReport getReport() {
  YarnClient yarnClient = createYarnClient();
  try {
    return new Hadoop21YarnApplicationReport(yarnClient.getApplicationReport(appId));
  } catch (YarnException | IOException e) {
    throw new RuntimeException("Failed to get application report for " + appId, e);
  } finally {
    yarnClient.stop();
  }
}
Example 13
Source File: Hadoop21YarnAppClient.java    From twill with Apache License 2.0 5 votes vote down vote up 
@Override
public List<NodeReport> getNodeReports() throws Exception {
  YarnClient yarnClient = createYarnClient();
  try {
    return yarnClient.getNodeReports();
  } finally {
    yarnClient.stop();
  }
}
Example 14
Source File: TestYarnClient.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 10000)
public void testGetContainerReport() throws YarnException, IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
      true);
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports = ((MockYarnClient) client)
      .getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  ContainerReport report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 1)).toString());
  containerId = ContainerId.newContainerId(appAttemptId, 3);
  report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 3)).toString());
  client.stop();
}
Example 15
Source File: Client.java    From stratosphere with Apache License 2.0 5 votes vote down vote up 
private void showClusterMetrics(YarnClient yarnClient)
		throws YarnException, IOException {
	YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();
	System.out.println("NodeManagers in Cluster " + metrics.getNumNodeManagers());
	List<NodeReport> nodes = yarnClient.getNodeReports();
	final String format = "|%-16s |%-16s %n";
	System.out.printf("|Property         |Value          %n");
	System.out.println("+---------------------------------------+");
	int totalMemory = 0;
	int totalCores = 0;
	for(NodeReport rep : nodes) {
		final Resource res = rep.getCapability();
		totalMemory += res.getMemory();
		totalCores += res.getVirtualCores();
		System.out.format(format, "NodeID", rep.getNodeId());
		System.out.format(format, "Memory", res.getMemory()+" MB");
		System.out.format(format, "vCores", res.getVirtualCores());
		System.out.format(format, "HealthReport", rep.getHealthReport());
		System.out.format(format, "Containers", rep.getNumContainers());
		System.out.println("+---------------------------------------+");
	}
	System.out.println("Summary: totalMemory "+totalMemory+" totalCores "+totalCores);
	List<QueueInfo> qInfo = yarnClient.getAllQueues();
	for(QueueInfo q : qInfo) {
		System.out.println("Queue: "+q.getQueueName()+", Current Capacity: "+q.getCurrentCapacity()+" Max Capacity: "+q.getMaximumCapacity()+" Applications: "+q.getApplications().size());
	}
	yarnClient.stop();
	System.exit(0);
}
Example 16
Source File: TestYarnClient.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testClientStop() {
  Configuration conf = new Configuration();
  ResourceManager rm = new ResourceManager();
  rm.init(conf);
  rm.start();

  YarnClient client = YarnClient.createYarnClient();
  client.init(conf);
  client.start();
  client.stop();
  rm.stop();
}
Example 17
Source File: AbstractYarnClusterDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Kills YARN application and stops YARN client.
 *
 * <p>Use this method to kill the App before it has been properly deployed
 */
private void failSessionDuringDeployment(YarnClient yarnClient, YarnClientApplication yarnApplication) {
	LOG.info("Killing YARN application");

	try {
		yarnClient.killApplication(yarnApplication.getNewApplicationResponse().getApplicationId());
	} catch (Exception e) {
		// we only log a debug message here because the "killApplication" call is a best-effort
		// call (we don't know if the application has been deployed when the error occured).
		LOG.debug("Error while killing YARN application", e);
	}
	yarnClient.stop();
}
Example 18
Source File: BroadcastAndOneToOneExample.java    From tez with Apache License 2.0 4 votes vote down vote up 
private DAG createDAG(FileSystem fs, TezConfiguration tezConf,
    Path stagingDir, boolean doLocalityCheck) throws IOException, YarnException {

  int numBroadcastTasks = 2;
  int numOneToOneTasks = 3;
  if (doLocalityCheck) {
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(tezConf);
    yarnClient.start();
    int numNMs = yarnClient.getNodeReports(NodeState.RUNNING).size();
    yarnClient.stop();
    // create enough 1-1 tasks to run in parallel
    numOneToOneTasks = numNMs - numBroadcastTasks - 1;// 1 AM
    if (numOneToOneTasks < 1) {
      numOneToOneTasks = 1;
    }
  }
  byte[] procByte = {(byte) (doLocalityCheck ? 1 : 0), 1};
  UserPayload procPayload = UserPayload.create(ByteBuffer.wrap(procByte));

  System.out.println("Using " + numOneToOneTasks + " 1-1 tasks");

  Vertex broadcastVertex = Vertex.create("Broadcast", ProcessorDescriptor.create(
      InputProcessor.class.getName()), numBroadcastTasks);
  
  Vertex inputVertex = Vertex.create("Input", ProcessorDescriptor.create(
      InputProcessor.class.getName()).setUserPayload(procPayload), numOneToOneTasks);

  Vertex oneToOneVertex = Vertex.create("OneToOne",
      ProcessorDescriptor.create(
          OneToOneProcessor.class.getName()).setUserPayload(procPayload));
  oneToOneVertex.setVertexManagerPlugin(
      VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName()));

  UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
      .newBuilder(Text.class.getName(), IntWritable.class.getName())
      .setFromConfiguration(tezConf).build();

  DAG dag = DAG.create("BroadcastAndOneToOneExample");
  dag.addVertex(inputVertex)
      .addVertex(broadcastVertex)
      .addVertex(oneToOneVertex)
      .addEdge(
          Edge.create(inputVertex, oneToOneVertex, edgeConf.createDefaultOneToOneEdgeProperty()))
      .addEdge(
          Edge.create(broadcastVertex, oneToOneVertex,
              edgeConf.createDefaultBroadcastEdgeProperty()));
  return dag;
}
Example 19
Source File: TestYarnClient.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
Example 20
Source File: TestYarnClient.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}