org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl Java Examples

@Inject
private YarnContainerManager(
    @Parameter(YarnHeartbeatPeriod.class) final int yarnRMHeartbeatPeriod,
    @Parameter(JobSubmissionDirectory.class) final String jobSubmissionDirectory,
    final YarnConfiguration yarnConf,
    final YarnProxyUser yarnProxyUser,
    final REEFEventHandlers reefEventHandlers,
    final Containers containers,
    final ApplicationMasterRegistration registration,
    final ContainerRequestCounter containerRequestCounter,
    final DriverStatusManager driverStatusManager,
    final REEFFileNames reefFileNames,
    final TrackingURLProvider trackingURLProvider,
    final LocalAddressProvider addressProvider,
    final RackNameFormatter rackNameFormatter,
    final InjectionFuture<ProgressProvider> progressProvider) throws IOException {

  this.reefEventHandlers = reefEventHandlers;
  this.driverStatusManager = driverStatusManager;

  this.containers = containers;
  this.registration = registration;
  this.containerRequestCounter = containerRequestCounter;
  this.yarnConf = yarnConf;
  this.yarnProxyUser = yarnProxyUser;
  this.rackNameFormatter = rackNameFormatter;

  this.trackingUrl = trackingURLProvider.getTrackingUrl();
  this.amRegistrationHost = addressProvider.getLocalAddress();

  this.resourceManager = AMRMClientAsync.createAMRMClientAsync(yarnRMHeartbeatPeriod, this);
  this.nodeManager = new NMClientAsyncImpl(this);

  this.jobSubmissionDirectory = jobSubmissionDirectory;
  this.reefFileNames = reefFileNames;
  this.progressProvider = progressProvider;

  LOG.log(Level.INFO, "Instantiated YarnContainerManager: {0} {1}, trackingUrl: {3}, jobSubmissionDirectory: {4}.",
      new Object[] {this.registration, this.yarnProxyUser, this.trackingUrl, this.jobSubmissionDirectory});
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception
{
  LOG.info("Application master" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId());

  FileInputStream fis = new FileInputStream("./" + LogicalPlan.SER_FILE_NAME);
  try {
    this.dag = LogicalPlan.read(fis);
  } finally {
    fis.close();
  }
  // "debug" simply dumps all data using LOG.info
  if (dag.isDebug()) {
    dumpOutDebugInfo();
  }
  dag.setAttribute(LogicalPlan.APPLICATION_ATTEMPT_ID, appAttemptID.getAttemptId());
  FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), conf);
  this.dnmgr = StreamingContainerManager.getInstance(recoveryHandler, dag, true);
  dag = this.dnmgr.getLogicalPlan();
  this.appContext = new ClusterAppContextImpl(dag.getAttributes());

  Map<Class<?>, Class<? extends StringCodec<?>>> codecs = dag.getAttributes().get(DAG.STRING_CODECS);
  StringCodecs.loadConverters(codecs);

  LOG.info("Starting application with {} operators in {} containers", dnmgr.getPhysicalPlan().getAllOperators().size(), dnmgr.getPhysicalPlan().getContainers().size());

  // Setup security configuration such as that for web security
  SecurityUtils.init(conf, dag.getValue(LogicalPlan.STRAM_HTTP_AUTHENTICATION));

  if (UserGroupInformation.isSecurityEnabled()) {
    // TODO :- Need to perform token renewal
    delegationTokenManager = new StramDelegationTokenManager(DELEGATION_KEY_UPDATE_INTERVAL, DELEGATION_TOKEN_MAX_LIFETIME, DELEGATION_TOKEN_RENEW_INTERVAL, DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL);
  }
  this.nmClient = new NMClientAsyncImpl(new NMCallbackHandler());
  addService(nmClient);
  this.amRmClient = AMRMClient.createAMRMClient();
  addService(amRmClient);

  // start RPC server
  int rpcListenerCount = dag.getValue(DAGContext.HEARTBEAT_LISTENER_THREAD_COUNT);
  this.heartbeatListener = new StreamingContainerParent(this.getClass().getName(), dnmgr, delegationTokenManager, rpcListenerCount);
  addService(heartbeatListener);

  AutoMetric.Transport appDataPushTransport = dag.getValue(LogicalPlan.METRICS_TRANSPORT);
  if (appDataPushTransport != null) {
    this.appDataPushAgent = new AppDataPushAgent(dnmgr, appContext);
    addService(this.appDataPushAgent);
  }
  initApexPluginDispatcher();

  // Initialize all services added above
  super.serviceInit(conf);
}
 

private void setupNMConnection() {
  containerListener = new NMCallbackHandler(this);
  nmClientAsync = new NMClientAsyncImpl(containerListener);
  nmClientAsync.init(conf);
  nmClientAsync.start();
}
 

public static NMClientAsync createNMClientAsync(
    CallbackHandler callbackHandler) {
  return new NMClientAsyncImpl(callbackHandler);
}
 

public static NMClientAsync createNMClientAsync(
    CallbackHandler callbackHandler) {
  return new NMClientAsyncImpl(callbackHandler);
}
 

@SuppressWarnings({ "unchecked" })
public void run() throws YarnException, IOException, InterruptedException {
  LOG.info("Starting ApplicationMaster");

  // Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class
  // are marked as LimitedPrivate
  Credentials credentials =
      UserGroupInformation.getCurrentUser().getCredentials();
  DataOutputBuffer dob = new DataOutputBuffer();
  credentials.writeTokenStorageToStream(dob);
  // Now remove the AM->RM token so that containers cannot access it.
  Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
  LOG.info("Executing with tokens:");
  while (iter.hasNext()) {
    Token<?> token = iter.next();
    LOG.info(token);
    if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
      iter.remove();
    }
  }
  allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

  // Create appSubmitterUgi and add original tokens to it
  String appSubmitterUserName =
      System.getenv(ApplicationConstants.Environment.USER.name());
  appSubmitterUgi =
      UserGroupInformation.createRemoteUser(appSubmitterUserName);
  appSubmitterUgi.addCredentials(credentials);


  AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
  amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
  amRMClient.init(conf);
  amRMClient.start();

  containerListener = createNMCallbackHandler();
  nmClientAsync = new NMClientAsyncImpl(containerListener);
  nmClientAsync.init(conf);
  nmClientAsync.start();

  startTimelineClient(conf);
  if(timelineClient != null) {
    publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
        DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
  }

  // Setup local RPC Server to accept status requests directly from clients
  // TODO need to setup a protocol for client to be able to communicate to
  // the RPC server
  // TODO use the rpc port info to register with the RM for the client to
  // send requests to this app master

  // Register self with ResourceManager
  // This will start heartbeating to the RM
  appMasterHostname = NetUtils.getHostname();
  RegisterApplicationMasterResponse response = amRMClient
      .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
          appMasterTrackingUrl);
  // Dump out information about cluster capability as seen by the
  // resource manager
  int maxMem = response.getMaximumResourceCapability().getMemory();
  LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
  
  int maxVCores = response.getMaximumResourceCapability().getVirtualCores();
  LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores);

  // A resource ask cannot exceed the max.
  if (containerMemory > maxMem) {
    LOG.info("Container memory specified above max threshold of cluster."
        + " Using max value." + ", specified=" + containerMemory + ", max="
        + maxMem);
    containerMemory = maxMem;
  }

  if (containerVirtualCores > maxVCores) {
    LOG.info("Container virtual cores specified above max threshold of cluster."
        + " Using max value." + ", specified=" + containerVirtualCores + ", max="
        + maxVCores);
    containerVirtualCores = maxVCores;
  }

  List<Container> previousAMRunningContainers =
      response.getContainersFromPreviousAttempts();
  LOG.info(appAttemptID + " received " + previousAMRunningContainers.size()
    + " previous attempts' running containers on AM registration.");
  numAllocatedContainers.addAndGet(previousAMRunningContainers.size());

  int numTotalContainersToRequest =
      numTotalContainers - previousAMRunningContainers.size();
  // Setup ask for containers from RM
  // Send request for containers to RM
  // Until we get our fully allocated quota, we keep on polling RM for
  // containers
  // Keep looping until all the containers are launched and shell script
  // executed on them ( regardless of success/failure).
  for (int i = 0; i < numTotalContainersToRequest; ++i) {
    ContainerRequest containerAsk = setupContainerAskForRM();
    amRMClient.addContainerRequest(containerAsk);
  }
  numRequestedContainers.set(numTotalContainers);
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception
{
  LOG.info("Application master" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId());

  FileInputStream fis = new FileInputStream("./" + LogicalPlan.SER_FILE_NAME);
  try {
    this.dag = LogicalPlan.read(fis);
  } finally {
    fis.close();
  }
  // "debug" simply dumps all data using LOG.info
  if (dag.isDebug()) {
    dumpOutDebugInfo();
  }
  dag.setAttribute(LogicalPlan.APPLICATION_ATTEMPT_ID, appAttemptID.getAttemptId());
  FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), conf);
  this.dnmgr = StreamingContainerManager.getInstance(recoveryHandler, dag, true);
  dag = this.dnmgr.getLogicalPlan();
  this.appContext = new ClusterAppContextImpl(dag.getAttributes());

  Map<Class<?>, Class<? extends StringCodec<?>>> codecs = dag.getAttributes().get(DAG.STRING_CODECS);
  StringCodecs.loadConverters(codecs);

  LOG.info("Starting application with {} operators in {} containers", dnmgr.getPhysicalPlan().getAllOperators().size(), dnmgr.getPhysicalPlan().getContainers().size());

  // Setup security configuration such as that for web security
  SecurityUtils.init(conf, dag.getValue(LogicalPlan.STRAM_HTTP_AUTHENTICATION));

  if (UserGroupInformation.isSecurityEnabled()) {
    // TODO :- Need to perform token renewal
    delegationTokenManager = new StramDelegationTokenManager(DELEGATION_KEY_UPDATE_INTERVAL, DELEGATION_TOKEN_MAX_LIFETIME, DELEGATION_TOKEN_RENEW_INTERVAL, DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL);
  }
  this.nmClient = new NMClientAsyncImpl(new NMCallbackHandler());
  addService(nmClient);
  this.amRmClient = AMRMClient.createAMRMClient();
  addService(amRmClient);

  // start RPC server
  int rpcListenerCount = dag.getValue(DAGContext.HEARTBEAT_LISTENER_THREAD_COUNT);
  this.heartbeatListener = new StreamingContainerParent(this.getClass().getName(), dnmgr, delegationTokenManager, rpcListenerCount);
  addService(heartbeatListener);

  AutoMetric.Transport appDataPushTransport = dag.getValue(LogicalPlan.METRICS_TRANSPORT);
  if (appDataPushTransport != null) {
    this.appDataPushAgent = new AppDataPushAgent(dnmgr, appContext);
    addService(this.appDataPushAgent);
  }
  initApexPluginDispatcher();

  // Initialize all services added above
  super.serviceInit(conf);
}
 

@Test
public void testAmShutdown() throws IOException, YarnException {

  Assume.assumeTrue(
      "This test requires a YARN Resource Manager to connect to",
      Boolean.parseBoolean(System.getenv("REEF_TEST_YARN")));

  final YarnConfiguration yarnConfig = new YarnConfiguration();

  // Start YARN client and register the application

  final YarnClient yarnClient = YarnClient.createYarnClient();
  yarnClient.init(yarnConfig);
  yarnClient.start();

  final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class);
  containerContext.setCommands(Collections.<String>emptyList());
  containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap());
  containerContext.setEnvironment(Collections.<String, String>emptyMap());
  containerContext.setTokens(getTokens());

  final ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext();
  appContext.setApplicationName("REEF_Unmanaged_AM_Test");
  appContext.setAMContainerSpec(containerContext);
  appContext.setUnmanagedAM(true);
  appContext.setQueue("default");

  final ApplicationId applicationId = appContext.getApplicationId();
  LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId);

  yarnClient.submitApplication(appContext);

  LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId);

  addToken(yarnClient.getAMRMToken(applicationId));

  // Start the AM

  final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this);
  rmClient.init(yarnConfig);
  rmClient.start();

  final NMClientAsync nmClient = new NMClientAsyncImpl(this);
  nmClient.init(yarnConfig);
  nmClient.start();

  final RegisterApplicationMasterResponse registration =
      rmClient.registerApplicationMaster(NetUtils.getHostname(), -1, null);

  LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration);

  rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null);

  LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState());

  // Shutdown the AM

  rmClient.stop();
  nmClient.stop();

  // Get the final application report

  final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);
  final YarnApplicationState appState = appReport.getYarnApplicationState();
  final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus();

  LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] {
      applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus});

  Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState);
  Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus);

  // Shutdown YARN client

  yarnClient.stop();
}