org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse Java Examples

@Override
protected void startUp() throws Exception {
  LOGGER.info("Starting the YarnService");

  // Register itself with the EventBus for container-related requests
  this.eventBus.register(this);

  this.amrmClientAsync.start();
  this.nmClientAsync.start();

  // The ApplicationMaster registration response is used to determine the maximum resource capacity of the cluster
  RegisterApplicationMasterResponse response = this.amrmClientAsync.registerApplicationMaster(
      GobblinClusterUtils.getHostname(), -1, "");
  LOGGER.info("ApplicationMaster registration response: " + response);
  this.maxResourceCapacity = Optional.of(response.getMaximumResourceCapability());

  LOGGER.info("Requesting initial containers");
  requestInitialContainers(this.initialContainers);
}
 

private void updateMatchingStrategy(final RegisterApplicationMasterResponse registerApplicationMasterResponse) {
	final Optional<Set<String>> schedulerResourceTypesOptional =
		registerApplicationMasterResponseReflector.getSchedulerResourceTypeNames(registerApplicationMasterResponse);

	final WorkerSpecContainerResourceAdapter.MatchingStrategy strategy;
	if (schedulerResourceTypesOptional.isPresent()) {
		Set<String> types = schedulerResourceTypesOptional.get();
		log.info("Register application master response contains scheduler resource types: {}.", types);
		matchingStrategy = types.contains("CPU") ?
			WorkerSpecContainerResourceAdapter.MatchingStrategy.MATCH_VCORE :
			WorkerSpecContainerResourceAdapter.MatchingStrategy.IGNORE_VCORE;
	} else {
		log.info("Register application master response does not contain scheduler resource types, use '{}'.",
			YarnConfigOptionsInternal.MATCH_CONTAINER_VCORES.key());
	}
	log.info("Container matching strategy: {}.", matchingStrategy);
}
 

@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  resetStartFailoverFlag(true);
  // make sure failover has been triggered
  Assert.assertTrue(waittingForFailOver());
  return createFakeRegisterApplicationMasterResponse();
}
 

@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  RegisterApplicationMasterRequestProto requestProto =
      ((RegisterApplicationMasterRequestPBImpl) request).getProto();
  try {
    return new RegisterApplicationMasterResponsePBImpl(
      proxy.registerApplicationMaster(null, requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 

@Override
public void init(CommandLine cliParser) throws Exception {
  LOG.info("Starting ApplicationMaster");

  // Args have to be initialized first
  this.args = new ApplicationMasterArgs(cliParser);
  this.clusterSpec = ClusterSpec.empty();

  String hostname = System.getenv(Environment.NM_HOST.name());
  int rpcPort = setupRPCService(hostname);
  RegisterApplicationMasterResponse response = setupRMConnection(hostname, rpcPort);
  setupPreviousRunningContainers(response);
  setupContainerResource(response);
  setupNMConnection();
}
 

@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  // TODO Auto-generated method stub
  return null;
}
 

private void getContainersFromPreviousAttempts(final RegisterApplicationMasterResponse registerApplicationMasterResponse) {
	final List<Container> containersFromPreviousAttempts =
		new RegisterApplicationMasterResponseReflector(log).getContainersFromPreviousAttempts(registerApplicationMasterResponse);

	log.info("Recovered {} containers from previous attempts ({}).", containersFromPreviousAttempts.size(), containersFromPreviousAttempts);

	for (final Container container : containersFromPreviousAttempts) {
		workerNodeMap.put(new ResourceID(container.getId().toString()), new YarnWorkerNode(container));
	}
}
 

@Test(timeout = 15000)
public void testRegisterApplicationMasterOnHA() throws YarnException,
    IOException {
  RegisterApplicationMasterRequest request =
      RegisterApplicationMasterRequest.newInstance("localhost", 0, "");
  RegisterApplicationMasterResponse response =
      amClient.registerApplicationMaster(request);
  Assert.assertEquals(response,
      this.cluster.createFakeRegisterApplicationMasterResponse());
}
 

protected void register() {
  //Register
  InetSocketAddress serviceAddr = null;
  if (clientService != null ) {
    serviceAddr = clientService.getBindAddress();
  }
  try {
    RegisterApplicationMasterRequest request =
      recordFactory.newRecordInstance(RegisterApplicationMasterRequest.class);
    if (serviceAddr != null) {
      request.setHost(serviceAddr.getHostName());
      request.setRpcPort(serviceAddr.getPort());
      request.setTrackingUrl(MRWebAppUtil
          .getAMWebappScheme(getConfig())
          + serviceAddr.getHostName() + ":" + clientService.getHttpPort());
    }
    RegisterApplicationMasterResponse response =
      scheduler.registerApplicationMaster(request);
    isApplicationMasterRegistered = true;
    maxContainerCapability = response.getMaximumResourceCapability();
    this.context.getClusterInfo().setMaxContainerCapability(
        maxContainerCapability);
    if (UserGroupInformation.isSecurityEnabled()) {
      setClientToAMToken(response.getClientToAMTokenMasterKey());        
    }
    this.applicationACLs = response.getApplicationACLs();
    LOG.info("maxContainerCapability: " + maxContainerCapability);
    String queue = response.getQueue();
    LOG.info("queue: " + queue);
    job.setQueueName(queue);
    this.schedulerResourceTypes.addAll(response.getSchedulerResourceTypes());
  } catch (Exception are) {
    LOG.error("Exception while registering", are);
    throw new YarnRuntimeException(are);
  }
}
 

@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  return RegisterApplicationMasterResponse.newInstance(
      Resource.newInstance(512, 1),
      Resource.newInstance(512000, 1024),
      Collections.<ApplicationAccessType,String>emptyMap(),
      ByteBuffer.wrap("fake_key".getBytes()),
      Collections.<Container>emptyList(),
      "default",
      Collections.<NMToken>emptyList());
}
 

@Override
protected void startUp() throws Exception {
  Preconditions.checkNotNull(trackerAddr, "Tracker address not set.");
  Preconditions.checkNotNull(trackerUrl, "Tracker URL not set.");

  amrmClient.start();
  RegisterApplicationMasterResponse response = amrmClient.registerApplicationMaster(trackerAddr.getHostName(),
                                                                                    trackerAddr.getPort(),
                                                                                    trackerUrl.toString());
  maxCapability = response.getMaximumResourceCapability();
  nmClient.startAndWait();
}
 

@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  RegisterApplicationMasterRequestProto requestProto =
      ((RegisterApplicationMasterRequestPBImpl) request).getProto();
  try {
    return new RegisterApplicationMasterResponsePBImpl(
      proxy.registerApplicationMaster(null, requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 

@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  // TODO Auto-generated method stub
  return null;
}
 

public RegisterApplicationMasterResponse registerAppAttempt(boolean wait)
    throws Exception {
  if (wait) {
    waitForState(RMAppAttemptState.LAUNCHED);
  }
  responseId = 0;
  final RegisterApplicationMasterRequest req =
      Records.newRecord(RegisterApplicationMasterRequest.class);
  req.setHost("");
  req.setRpcPort(1);
  req.setTrackingUrl("");
  if (ugi == null) {
    ugi = UserGroupInformation.createRemoteUser(
        attemptId.toString());
    Token<AMRMTokenIdentifier> token =
        context.getRMApps().get(attemptId.getApplicationId())
            .getRMAppAttempt(attemptId).getAMRMToken();
    ugi.addTokenIdentifier(token.decodeIdentifier());
  }
  try {
    return ugi
      .doAs(
          new PrivilegedExceptionAction<RegisterApplicationMasterResponse>() {
        @Override
        public RegisterApplicationMasterResponse run() throws Exception {
          return amRMProtocol.registerApplicationMaster(req);
        }
      });
  } catch (UndeclaredThrowableException e) {
    throw (Exception) e.getCause();
  }
}
 

@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    String appHostName, int appHostPort, String appTrackingUrl)
    throws YarnException, IOException {
  this.appHostName = appHostName;
  this.appHostPort = appHostPort;
  this.appTrackingUrl = appTrackingUrl;
  Preconditions.checkArgument(appHostName != null,
      "The host name should not be null");
  Preconditions.checkArgument(appHostPort >= -1, "Port number of the host"
      + " should be any integers larger than or equal to -1");

  return registerApplicationMaster();
}
 

private RegisterApplicationMasterResponse registerApplicationMaster()
    throws YarnException, IOException {
  RegisterApplicationMasterRequest request =
      RegisterApplicationMasterRequest.newInstance(this.appHostName,
          this.appHostPort, this.appTrackingUrl);
  RegisterApplicationMasterResponse response =
      rmClient.registerApplicationMaster(request);
  synchronized (this) {
    lastResponseId = 0;
    if (!response.getNMTokensFromPreviousAttempts().isEmpty()) {
      populateNMTokens(response.getNMTokensFromPreviousAttempts());
    }
  }
  return response;
}
 

/**
 * Registers this application master with the resource manager. On successful
 * registration, starts the heartbeating thread.
 * @throws YarnException
 * @throws IOException
 */
public RegisterApplicationMasterResponse registerApplicationMaster(
    String appHostName, int appHostPort, String appTrackingUrl)
    throws YarnException, IOException {
  RegisterApplicationMasterResponse response = client
      .registerApplicationMaster(appHostName, appHostPort, appTrackingUrl);
  heartbeatThread.start();
  return response;
}
 

private void getContainersFromPreviousAttempts(final RegisterApplicationMasterResponse registerApplicationMasterResponse) {
	final List<Container> containersFromPreviousAttempts =
		new RegisterApplicationMasterResponseReflector(log).getContainersFromPreviousAttempts(registerApplicationMasterResponse);

	log.info("Recovered {} containers from previous attempts ({}).", containersFromPreviousAttempts.size(), containersFromPreviousAttempts);

	for (final Container container : containersFromPreviousAttempts) {
		workerNodeMap.put(new ResourceID(container.getId().toString()), new YarnWorkerNode(container));
	}
}
 

public RegisterApplicationMasterResponse
createFakeRegisterApplicationMasterResponse() {
  Resource minCapability = Resource.newInstance(2048, 2);
  Resource maxCapability = Resource.newInstance(4096, 4);
  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>();
  acls.put(ApplicationAccessType.MODIFY_APP, "*");
  ByteBuffer key = ByteBuffer.wrap("fake_key".getBytes());
  return RegisterApplicationMasterResponse.newInstance(minCapability,
      maxCapability, acls, key, new ArrayList<Container>(), "root",
      new ArrayList<NMToken>());
}
 

@Test(timeout = 15000)
public void testRegisterApplicationMasterOnHA() throws YarnException,
    IOException {
  RegisterApplicationMasterRequest request =
      RegisterApplicationMasterRequest.newInstance("localhost", 0, "");
  RegisterApplicationMasterResponse response =
      amClient.registerApplicationMaster(request);
  Assert.assertEquals(response,
      this.cluster.createFakeRegisterApplicationMasterResponse());
}
 

protected void register() {
  //Register
  InetSocketAddress serviceAddr = null;
  if (clientService != null ) {
    serviceAddr = clientService.getBindAddress();
  }
  try {
    RegisterApplicationMasterRequest request =
      recordFactory.newRecordInstance(RegisterApplicationMasterRequest.class);
    if (serviceAddr != null) {
      request.setHost(serviceAddr.getHostName());
      request.setRpcPort(serviceAddr.getPort());
      request.setTrackingUrl(MRWebAppUtil
          .getAMWebappScheme(getConfig())
          + serviceAddr.getHostName() + ":" + clientService.getHttpPort());
    }
    RegisterApplicationMasterResponse response =
      scheduler.registerApplicationMaster(request);
    isApplicationMasterRegistered = true;
    maxContainerCapability = response.getMaximumResourceCapability();
    this.context.getClusterInfo().setMaxContainerCapability(
        maxContainerCapability);
    if (UserGroupInformation.isSecurityEnabled()) {
      setClientToAMToken(response.getClientToAMTokenMasterKey());        
    }
    this.applicationACLs = response.getApplicationACLs();
    LOG.info("maxContainerCapability: " + maxContainerCapability);
    String queue = response.getQueue();
    LOG.info("queue: " + queue);
    job.setQueueName(queue);
    this.schedulerResourceTypes.addAll(response.getSchedulerResourceTypes());
  } catch (Exception are) {
    LOG.error("Exception while registering", are);
    throw new YarnRuntimeException(are);
  }
}
 

@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  return RegisterApplicationMasterResponse.newInstance(
      Resource.newInstance(512, 1),
      Resource.newInstance(512000, 1024),
      Collections.<ApplicationAccessType,String>emptyMap(),
      ByteBuffer.wrap("fake_key".getBytes()),
      Collections.<Container>emptyList(),
      "default",
      Collections.<NMToken>emptyList());
}
 

private void getContainersFromPreviousAttempts(final RegisterApplicationMasterResponse registerApplicationMasterResponse) {
	final List<Container> containersFromPreviousAttempts =
		registerApplicationMasterResponseReflector.getContainersFromPreviousAttempts(registerApplicationMasterResponse);

	log.info("Recovered {} containers from previous attempts ({}).", containersFromPreviousAttempts.size(), containersFromPreviousAttempts);

	for (final Container container : containersFromPreviousAttempts) {
		workerNodeMap.put(new ResourceID(container.getId().toString()), new YarnWorkerNode(container));
	}
}
 

/**
 * Set the registration information. This is a set-once field.
 *
 * @param registration
 */
synchronized void setRegistration(final RegisterApplicationMasterResponse registration) {
  if (this.isPresent()) {
    throw new RuntimeException("Trying to re-register the AM");
  }
  this.registration = Optional.of(registration);
}
 

@SuppressWarnings("unchecked")
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    String appHostName, int appHostPort, String appTrackingUrl) {
  RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class);
  Resource mockMaxResource = mock(Resource.class);
  Map<ApplicationAccessType, String> mockAcls = mock(Map.class);
  when(mockRegResponse.getMaximumResourceCapability()).thenReturn(
      mockMaxResource);
  when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
  return mockRegResponse;
}
 

@Inject
public void onInit(VMConfig vmConfig) throws Exception {
  logger.info("Start init(VMConfig vmConfig)");
  this.vmConfig = vmConfig;
  try {
    this.yarnConfig = vmConfig.getHadoopProperties();
    conf = new YarnConfiguration() ;
    vmConfig.overrideHadoopConfiguration(conf);
    
    amrmClient = AMRMClient.createAMRMClient();
    amrmClientAsync = AMRMClientAsync.createAMRMClientAsync(amrmClient, 1000, new AMRMCallbackHandler());
    amrmClientAsync.init(conf);
    amrmClientAsync.start();
    
    nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();
    // Register with RM
    String appHostName = InetAddress.getLocalHost().getHostAddress()  ;

    RegisterApplicationMasterResponse registerResponse = amrmClientAsync.registerApplicationMaster(appHostName, 0, "");
    System.out.println("amrmClientAsync.registerApplicationMaster");
  } catch(Throwable t) {
    logger.error("Error: " , t);
    t.printStackTrace();
  }
  logger.info("Finish init(VMConfig vmConfig)");
}
 

@Override
public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
  EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes();
  Set<String> supportedTypes = new HashSet<String>();
  for (SchedulerResourceTypes resourceType : supportedResourceTypes) {
    supportedTypes.add(resourceType.name());
  }
  return supportedTypes;
}
 

@Override
public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
  EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes();
  Set<String> supportedTypes = new HashSet<String>();
  for (SchedulerResourceTypes resourceType : supportedResourceTypes) {
    supportedTypes.add(resourceType.name());
  }
  return supportedTypes;
}
 

@SuppressWarnings("unchecked")
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    String appHostName, int appHostPort, String appTrackingUrl) {
  mockRegResponse = mock(RegisterApplicationMasterResponse.class);
  Resource mockMaxResource = mock(Resource.class);
  Map<ApplicationAccessType, String> mockAcls = mock(Map.class);
  when(mockRegResponse.getMaximumResourceCapability()).thenReturn(
      mockMaxResource);
  when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
  return mockRegResponse;
}
 

@Override
public RegisterApplicationMasterResponse registerApplicationMaster(String appHostName, int appHostPort,
    String appTrackingUrl) {
  mockRegResponse = mock(RegisterApplicationMasterResponse.class);
  Resource mockMaxResource = Resources.createResource(1024*1024, 1024);
  Map<ApplicationAccessType, String> mockAcls = Collections.emptyMap();
  when(mockRegResponse.getMaximumResourceCapability()).thenReturn(
      mockMaxResource);
  when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
  when(mockRegResponse.getSchedulerResourceTypes()).thenReturn(
      EnumSet.of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU));
  return mockRegResponse;
}