Java Code Examples for org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse#getContainersFromPreviousAttempts()

private void setupPreviousRunningContainers(RegisterApplicationMasterResponse response) {
  String containerIdStr =
      System.getenv(Environment.CONTAINER_ID.name());
  ContainerId containerId = ContainerId.fromString(containerIdStr);
  appAttemptId = containerId.getApplicationAttemptId();
  List<Container> previousAMRunningContainers =
      response.getContainersFromPreviousAttempts();
  LOG.info(appAttemptId + " received " + previousAMRunningContainers.size()
      + " previous attempts' running containers on AM registration.");
  for (Container container : previousAMRunningContainers) {
    launchedContainers.add(container.getId());
  }
  allocatedContainerNum.addAndGet(previousAMRunningContainers.size());
}
 

@SuppressWarnings({ "unchecked" })
public void run() throws YarnException, IOException, InterruptedException {
  LOG.info("Starting ApplicationMaster");

  // Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class
  // are marked as LimitedPrivate
  Credentials credentials =
      UserGroupInformation.getCurrentUser().getCredentials();
  DataOutputBuffer dob = new DataOutputBuffer();
  credentials.writeTokenStorageToStream(dob);
  // Now remove the AM->RM token so that containers cannot access it.
  Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
  LOG.info("Executing with tokens:");
  while (iter.hasNext()) {
    Token<?> token = iter.next();
    LOG.info(token);
    if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
      iter.remove();
    }
  }
  allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

  // Create appSubmitterUgi and add original tokens to it
  String appSubmitterUserName =
      System.getenv(ApplicationConstants.Environment.USER.name());
  appSubmitterUgi =
      UserGroupInformation.createRemoteUser(appSubmitterUserName);
  appSubmitterUgi.addCredentials(credentials);


  AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
  amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
  amRMClient.init(conf);
  amRMClient.start();

  containerListener = createNMCallbackHandler();
  nmClientAsync = new NMClientAsyncImpl(containerListener);
  nmClientAsync.init(conf);
  nmClientAsync.start();

  startTimelineClient(conf);
  if(timelineClient != null) {
    publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
        DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
  }

  // Setup local RPC Server to accept status requests directly from clients
  // TODO need to setup a protocol for client to be able to communicate to
  // the RPC server
  // TODO use the rpc port info to register with the RM for the client to
  // send requests to this app master

  // Register self with ResourceManager
  // This will start heartbeating to the RM
  appMasterHostname = NetUtils.getHostname();
  RegisterApplicationMasterResponse response = amRMClient
      .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
          appMasterTrackingUrl);
  // Dump out information about cluster capability as seen by the
  // resource manager
  int maxMem = response.getMaximumResourceCapability().getMemory();
  LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
  
  int maxVCores = response.getMaximumResourceCapability().getVirtualCores();
  LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores);

  // A resource ask cannot exceed the max.
  if (containerMemory > maxMem) {
    LOG.info("Container memory specified above max threshold of cluster."
        + " Using max value." + ", specified=" + containerMemory + ", max="
        + maxMem);
    containerMemory = maxMem;
  }

  if (containerVirtualCores > maxVCores) {
    LOG.info("Container virtual cores specified above max threshold of cluster."
        + " Using max value." + ", specified=" + containerVirtualCores + ", max="
        + maxVCores);
    containerVirtualCores = maxVCores;
  }

  List<Container> previousAMRunningContainers =
      response.getContainersFromPreviousAttempts();
  LOG.info(appAttemptID + " received " + previousAMRunningContainers.size()
    + " previous attempts' running containers on AM registration.");
  numAllocatedContainers.addAndGet(previousAMRunningContainers.size());

  int numTotalContainersToRequest =
      numTotalContainers - previousAMRunningContainers.size();
  // Setup ask for containers from RM
  // Send request for containers to RM
  // Until we get our fully allocated quota, we keep on polling RM for
  // containers
  // Keep looping until all the containers are launched and shell script
  // executed on them ( regardless of success/failure).
  for (int i = 0; i < numTotalContainersToRequest; ++i) {
    ContainerRequest containerAsk = setupContainerAskForRM();
    amRMClient.addContainerRequest(containerAsk);
  }
  numRequestedContainers.set(numTotalContainers);
}