org.apache.hadoop.yarn.client.api.NMClient Java Examples

The following examples show how to use org.apache.hadoop.yarn.client.api.NMClient. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SolrMaster.java    From yarn-proto with Apache License 2.0 6 votes vote down vote up 
public SolrMaster(CommandLine cli) throws Exception {
  this.cli = cli;
  Configuration hadoopConf = new Configuration();
  if (cli.hasOption("conf")) {
    hadoopConf.addResource(new Path(cli.getOptionValue("conf")));
    hadoopConf.reloadConfiguration();
  }
  conf = new YarnConfiguration(hadoopConf);

  nmClient = NMClient.createNMClient();
  nmClient.init(conf);
  nmClient.start();
  numContainersToWaitFor = Integer.parseInt(cli.getOptionValue("nodes"));
  memory = Integer.parseInt(cli.getOptionValue("memory", "512"));
  port = Integer.parseInt(cli.getOptionValue("port"));
  nextPort = port;

  SecureRandom random = new SecureRandom();
  this.randomStopKey = new BigInteger(130, random).toString(32);

  this.inetAddresses = getMyInetAddresses();
}
Example #2
Source File: YarnResourceManagerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
TestingYarnResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		Configuration flinkConfig,
		Map<String, String> env,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		@Nullable String webInterfaceUrl,
		AMRMClientAsync<AMRMClient.ContainerRequest> mockResourceManagerClient,
		NMClient mockNMClient,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		flinkConfig,
		env,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		webInterfaceUrl,
		jobManagerMetricGroup);
	this.mockNMClient = mockNMClient;
	this.mockResourceManagerClient = mockResourceManagerClient;
}
Example #3
Source File: YarnManager.java    From Scribengin with GNU Affero General Public License v3.0 5 votes vote down vote up 
@Inject
public void onInit(VMConfig vmConfig) throws Exception {
  logger.info("Start init(VMConfig vmConfig)");
  this.vmConfig = vmConfig;
  try {
    this.yarnConfig = vmConfig.getHadoopProperties();
    conf = new YarnConfiguration() ;
    vmConfig.overrideHadoopConfiguration(conf);
    
    amrmClient = AMRMClient.createAMRMClient();
    amrmClientAsync = AMRMClientAsync.createAMRMClientAsync(amrmClient, 1000, new AMRMCallbackHandler());
    amrmClientAsync.init(conf);
    amrmClientAsync.start();
    
    nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();
    // Register with RM
    String appHostName = InetAddress.getLocalHost().getHostAddress()  ;

    RegisterApplicationMasterResponse registerResponse = amrmClientAsync.registerApplicationMaster(appHostName, 0, "");
    System.out.println("amrmClientAsync.registerApplicationMaster");
  } catch(Throwable t) {
    logger.error("Error: " , t);
    t.printStackTrace();
  }
  logger.info("Finish init(VMConfig vmConfig)");
}
Example #4
Source File: AbstractApplicationMaster.java    From Scribengin with GNU Affero General Public License v3.0 5 votes vote down vote up 
public boolean run() throws IOException, YarnException {
  // Initialize clients to RM and NMs.
  LOG.info("ApplicationMaster::run");
  AMRMClientAsync.CallbackHandler rmListener = new RMCallbackHandler();
  resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, rmListener);
  resourceManager.init(conf);
  resourceManager.start();
  
  nodeManager = NMClient.createNMClient();
  nodeManager.init(conf);
  nodeManager.start();
  
  // Register with RM
  resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort,
      appMasterTrackingUrl);

  Log.info("total container count: "+Integer.toString(totalContainerCount));
  
  // Ask RM to give us a bunch of containers
  //for (int i = 0; i < totalContainerCount; i++) {
    ContainerRequest containerReq = setupContainerReqForRM();
    resourceManager.addContainerRequest(containerReq);
  //}
  requestedContainerCount.addAndGet(totalContainerCount);

  while (!done) {
    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
    }
  }// while

  // Un-register with ResourceManager
  resourceManager.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
  return true;
}
Example #5
Source File: TestNMClientAsync.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private NMClient mockNMClient(int mode)
    throws YarnException, IOException {
  NMClient client = mock(NMClient.class);
  switch (mode) {
    case 0:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doNothing().when(client).stopContainer(any(ContainerId.class),
          any(NodeId.class));
      break;
    case 1:
      doThrow(RPCUtil.getRemoteException("Start Exception")).when(client)
          .startContainer(any(Container.class),
              any(ContainerLaunchContext.class));
      doThrow(RPCUtil.getRemoteException("Query Exception")).when(client)
          .getContainerStatus(any(ContainerId.class), any(NodeId.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
      break;
    case 2:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
  }
  return client;
}
Example #6
Source File: NMClientAsync.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Private
@VisibleForTesting
protected NMClientAsync(String name, NMClient client,
    CallbackHandler callbackHandler) {
  super(name);
  this.setClient(client);
  this.setCallbackHandler(callbackHandler);
}
Example #7
Source File: NMClientAsyncImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Private
@VisibleForTesting
protected NMClientAsyncImpl(String name, NMClient client,
    CallbackHandler callbackHandler) {
  super(name, client, callbackHandler);
  this.client = client;
  this.callbackHandler = callbackHandler;
}
Example #8
Source File: TestNMClientAsync.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private NMClient mockNMClient(int mode)
    throws YarnException, IOException {
  NMClient client = mock(NMClient.class);
  switch (mode) {
    case 0:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doNothing().when(client).stopContainer(any(ContainerId.class),
          any(NodeId.class));
      break;
    case 1:
      doThrow(RPCUtil.getRemoteException("Start Exception")).when(client)
          .startContainer(any(Container.class),
              any(ContainerLaunchContext.class));
      doThrow(RPCUtil.getRemoteException("Query Exception")).when(client)
          .getContainerStatus(any(ContainerId.class), any(NodeId.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
      break;
    case 2:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
  }
  return client;
}
Example #9
Source File: YarnResourceManager.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	// create the client to communicate with the node managers
	NMClient nodeManagerClient = NMClient.createNMClient();
	nodeManagerClient.init(yarnConfiguration);
	nodeManagerClient.start();
	nodeManagerClient.cleanupRunningContainersOnStop(true);
	return nodeManagerClient;
}
Example #10
Source File: NMClientAsync.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Private
@VisibleForTesting
protected NMClientAsync(String name, NMClient client,
    CallbackHandler callbackHandler) {
  super(name);
  this.setClient(client);
  this.setCallbackHandler(callbackHandler);
}
Example #11
Source File: YarnResourceManager.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	// create the client to communicate with the node managers
	NMClient nodeManagerClient = NMClient.createNMClient();
	nodeManagerClient.init(yarnConfiguration);
	nodeManagerClient.start();
	nodeManagerClient.cleanupRunningContainersOnStop(true);
	return nodeManagerClient;
}
Example #12
Source File: YarnResourceManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
TestingYarnResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		Configuration flinkConfig,
		Map<String, String> env,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		@Nullable String webInterfaceUrl,
		AMRMClientAsync<AMRMClient.ContainerRequest> mockResourceManagerClient,
		NMClient mockNMClient,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		flinkConfig,
		env,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		webInterfaceUrl,
		jobManagerMetricGroup);
	this.mockNMClient = mockNMClient;
	this.mockResourceManagerClient = mockResourceManagerClient;
}
Example #13
Source File: NMClientAsyncImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Private
@VisibleForTesting
protected NMClientAsyncImpl(String name, NMClient client,
    CallbackHandler callbackHandler) {
  super(name, client, callbackHandler);
  this.client = client;
  this.callbackHandler = callbackHandler;
}
Example #14
Source File: NMClientAsync.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public NMClient getClient() {
  return client;
}
Example #15
Source File: YarnResourceManagerTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	return mockNMClient;
}
Example #16
Source File: TestNMClient.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}
Example #17
Source File: NMClientAsync.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public void setClient(NMClient client) {
  this.client = client;
}
Example #18
Source File: NMClientAsync.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public NMClient getClient() {
  return client;
}
Example #19
Source File: YarnResourceManagerTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	return mockNMClient;
}
Example #20
Source File: Hadoop21YarnNMClient.java    From twill with Apache License 2.0 4 votes vote down vote up 
private ContainerTerminator(Container container, NMClient nmClient) {
  this.container = container;
  this.nmClient = nmClient;
}
Example #21
Source File: Hadoop21YarnNMClient.java    From twill with Apache License 2.0 4 votes vote down vote up 
public Hadoop21YarnNMClient(Configuration configuration) {
  this.nmClient = NMClient.createNMClient();
  nmClient.init(configuration);
}
Example #22
Source File: TestNMClient.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}
Example #23
Source File: NMClientAsync.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public void setClient(NMClient client) {
  this.client = client;
}
Example #24
Source File: ApplicationMaster.java    From ignite with Apache License 2.0 2 votes vote down vote up 
/**
 * Sets NMClient.
 *
 * @param nmClient NMClient.
 */
public void setNmClient(NMClient nmClient) {
    this.nmClient = nmClient;
}
Example #25
Source File: YarnManager.java    From Scribengin with GNU Affero General Public License v3.0 votes vote down vote up 
public NMClient getNMClient() { return this.nmClient ; }