org.apache.hadoop.yarn.client.api.async.NMClientAsync Java Examples

@Test
public void testAllocatedResourceExpiryForYarn() {
  YarnConfiguration yarnConfiguration = mock(YarnConfiguration.class);
  SamzaAppMasterMetrics metrics = mock(SamzaAppMasterMetrics.class);
  Config config = mock(Config.class);
  AMRMClientAsync asyncClient = mock(AMRMClientAsync.class);
  YarnAppState yarnAppState = new YarnAppState(0, mock(ContainerId.class), "host", 8080, 8081);
  SamzaYarnAppMasterLifecycle lifecycle = mock(SamzaYarnAppMasterLifecycle.class);
  SamzaYarnAppMasterService service = mock(SamzaYarnAppMasterService.class);
  NMClientAsync asyncNMClient = mock(NMClientAsync.class);
  ClusterResourceManager.Callback callback = mock(ClusterResourceManager.Callback.class);

  // start the cluster manager
  YarnClusterResourceManager yarnClusterResourceManager = new YarnClusterResourceManager(asyncClient, asyncNMClient,
      callback, yarnAppState, lifecycle, service, metrics, yarnConfiguration, config);

  SamzaResource allocatedResource = mock(SamzaResource.class);
  when(allocatedResource.getTimestamp()).thenReturn(System.currentTimeMillis() - Duration.ofMinutes(10).toMillis());

  Assert.assertTrue(yarnClusterResourceManager.isResourceExpired(allocatedResource));
}
 

public LaunchContainer( Configuration conf
        , String zkQuorum
        , String zkRoot
        , NMClientAsync nmClientAsync
        , ModelRequest request
        , Container container
        , ByteBuffer allTokens
        , Path appJarLocation
)
{
  this.allTokens = allTokens;
  this.zkQuorum = zkQuorum;
  this.zkRoot = zkRoot;
  this.request = request;
  this.container = container;
  this.conf = conf;
  this.nmClientAsync = nmClientAsync;
  this.appJarLocation = appJarLocation;
}
 

public void stop(NMClientAsync nmClient)
{
  if (stop == Long.MAX_VALUE) {
    nmClient.stopContainerAsync(container.getId(), container.getNodeId());
    LOG.info("Requested stop container {}", container.getId().toString());
    stop = System.currentTimeMillis();
  }
}
 

public StatefulContainer(NMClientAsync client, ContainerId containerId) {
  this.nmClientAsync = client;
  this.containerId = containerId;
  stateMachine = stateMachineFactory.make(this);
  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  readLock = lock.readLock();
  writeLock = lock.writeLock();
}
 

public StatefulContainer(NMClientAsync client, ContainerId containerId) {
  this.nmClientAsync = client;
  this.containerId = containerId;
  stateMachine = stateMachineFactory.make(this);
  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  readLock = lock.readLock();
  writeLock = lock.writeLock();
}
 

/**
 * @param lcontainer Allocated container
 * @param nmClient
 * @param sca
 * @param tokens
 */
public LaunchContainerRunnable(Container lcontainer, NMClientAsync nmClient, StreamingContainerAgent sca, ByteBuffer tokens)
{
  this.container = lcontainer;
  this.nmClient = nmClient;
  this.dag = sca.getContainer().getPlan().getLogicalPlan();
  this.tokens = tokens;
  this.sca = sca;
}
 

public void stop(NMClientAsync nmClient)
{
  if (stop == Long.MAX_VALUE) {
    nmClient.stopContainerAsync(container.getId(), container.getNodeId());
    LOG.info("Requested stop container {}", container.getId().toString());
    stop = System.currentTimeMillis();
  }
}
 

YarnClusterResourceManager(AMRMClientAsync amClientAsync, NMClientAsync nmClientAsync, Callback callback,
    YarnAppState yarnAppState, SamzaYarnAppMasterLifecycle lifecycle, SamzaYarnAppMasterService service,
    SamzaAppMasterMetrics metrics, YarnConfiguration yarnConfiguration, Config config) {
  super(callback);
  this.yarnConfiguration  = yarnConfiguration;
  this.metrics = metrics;
  this.yarnConfig = new YarnConfig(config);
  this.config = config;
  this.amClient = amClientAsync;
  this.state = yarnAppState;
  this.lifecycle = lifecycle;
  this.service = service;
  this.nmClientAsync = nmClientAsync;
}
 

/**
 * @param lcontainer Allocated container
 * @param nmClient
 * @param sca
 * @param tokens
 */
public LaunchContainerRunnable(Container lcontainer, NMClientAsync nmClient, StreamingContainerAgent sca, ByteBuffer tokens)
{
  this.container = lcontainer;
  this.nmClient = nmClient;
  this.dag = sca.getContainer().getPlan().getLogicalPlan();
  this.tokens = tokens;
  this.sca = sca;
}
 

@Test
public void testErrorInStartContainerShouldUpdateState() {
  // create mocks
  final int samzaContainerId = 1;
  YarnConfiguration yarnConfiguration = mock(YarnConfiguration.class);
  SamzaAppMasterMetrics metrics = mock(SamzaAppMasterMetrics.class);
  Config config = mock(Config.class);
  AMRMClientAsync asyncClient = mock(AMRMClientAsync.class);
  YarnAppState yarnAppState = new YarnAppState(0, mock(ContainerId.class), "host", 8080, 8081);
  SamzaYarnAppMasterLifecycle lifecycle = mock(SamzaYarnAppMasterLifecycle.class);
  SamzaYarnAppMasterService service = mock(SamzaYarnAppMasterService.class);
  NMClientAsync asyncNMClient = mock(NMClientAsync.class);
  ClusterResourceManager.Callback callback = mock(ClusterResourceManager.Callback.class);

  // start the cluster manager
  YarnClusterResourceManager yarnClusterResourceManager = new YarnClusterResourceManager(asyncClient, asyncNMClient,
      callback, yarnAppState, lifecycle, service, metrics, yarnConfiguration, config);

  yarnAppState.pendingProcessors.put(String.valueOf(samzaContainerId),
      new YarnContainer(Container.newInstance(
          ContainerId.newContainerId(
              ApplicationAttemptId.newInstance(
                  ApplicationId.newInstance(10000L, 1), 1), 1),
          NodeId.newInstance("host1", 8088), "http://host1",
          Resource.newInstance(1024, 1), Priority.newInstance(1),
          Token.newInstance("id".getBytes(), "read", "password".getBytes(), "service"))));

  yarnClusterResourceManager.start();
  assertEquals(1, yarnAppState.pendingProcessors.size());

  yarnClusterResourceManager.onStartContainerError(ContainerId.newContainerId(
      ApplicationAttemptId.newInstance(
          ApplicationId.newInstance(10000L, 1), 1), 1),
      new Exception());

  assertEquals(0, yarnAppState.pendingProcessors.size());
  verify(callback, times(1)).onStreamProcessorLaunchFailure(anyObject(), any(Exception.class));
}
 

public void initialize(AMRMClientAsync<AMRMClient.ContainerRequest> amRMClient
                      , NMClientAsync nmClient
                      , ServiceDiscoverer serviceDiscoverer
                      )
{
  this.nmClient = nmClient;
  this.amRMClient = amRMClient;
  this.serviceDiscoverer = serviceDiscoverer;
}
 

protected NMClientAsync createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	// create the client to communicate with the node managers
	NMClientAsync nodeManagerClient = NMClientAsync.createNMClientAsync(this);
	nodeManagerClient.init(yarnConfiguration);
	nodeManagerClient.start();
	return nodeManagerClient;
}
 

/**
 * Creates an YarnClusterResourceManager from config, a jobModelReader and a callback.
 * @param config to instantiate the cluster manager with
 * @param jobModelManager the jobModel manager to get the job model (mostly for the UI)
 * @param callback the callback to receive events from Yarn.
 * @param samzaAppState samza app state for display in the UI
 */
public YarnClusterResourceManager(Config config, JobModelManager jobModelManager,
    ClusterResourceManager.Callback callback, SamzaApplicationState samzaAppState) {
  super(callback);
  yarnConfiguration = new YarnConfiguration();
  yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

  // Use the Samza job config "fs.<scheme>.impl" and "fs.<scheme>.impl.*" for YarnConfiguration
  FileSystemImplConfig fsImplConfig = new FileSystemImplConfig(config);
  fsImplConfig.getSchemes().forEach(
    scheme -> {
      fsImplConfig.getSchemeConfig(scheme).forEach(
        (confKey, confValue) -> yarnConfiguration.set(confKey, confValue)
      );
    }
  );

  MetricsRegistryMap registry = new MetricsRegistryMap();
  metrics = new SamzaAppMasterMetrics(config, samzaAppState, registry);

  // parse configs from the Yarn environment
  String containerIdStr = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString());
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  String nodeHostString = System.getenv(ApplicationConstants.Environment.NM_HOST.toString());
  String nodePortString = System.getenv(ApplicationConstants.Environment.NM_PORT.toString());
  String nodeHttpPortString = System.getenv(ApplicationConstants.Environment.NM_HTTP_PORT.toString());

  int nodePort = Integer.parseInt(nodePortString);
  int nodeHttpPort = Integer.parseInt(nodeHttpPortString);
  YarnConfig yarnConfig = new YarnConfig(config);
  this.yarnConfig = yarnConfig;
  this.config = config;
  int interval = yarnConfig.getAMPollIntervalMs();

  //Instantiate the AM Client.
  this.amClient = AMRMClientAsync.createAMRMClientAsync(interval, this);

  this.state = new YarnAppState(-1, containerId, nodeHostString, nodePort, nodeHttpPort);

  log.info("Initialized YarnAppState: {}", state.toString());
  this.service = new SamzaYarnAppMasterService(config, samzaAppState, this.state, registry, yarnConfiguration);

  log.info("Container ID: {}, Nodehost:  {} , Nodeport : {} , NodeHttpport: {}", containerIdStr, nodeHostString, nodePort, nodeHttpPort);
  ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
  this.lifecycle = new SamzaYarnAppMasterLifecycle(
      clusterManagerConfig.getContainerMemoryMb(),
      clusterManagerConfig.getNumCores(),
      samzaAppState,
      state,
      amClient
  );
  this.nmClientAsync = NMClientAsync.createNMClientAsync(this);

}
 

public NMClientAsync getNmClientAsync() {
	return nmClientAsync;
}
 

public YarnService(Config config, String applicationName, String applicationId, YarnConfiguration yarnConfiguration,
    FileSystem fs, EventBus eventBus, HelixManager helixManager) throws Exception {
  this.applicationName = applicationName;
  this.applicationId = applicationId;

  this.config = config;

  this.eventBus = eventBus;

  this.helixManager = helixManager;

  this.gobblinMetrics = config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY) ?
      Optional.of(buildGobblinMetrics()) : Optional.<GobblinMetrics>absent();

  this.eventSubmitter = config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY) ?
      Optional.of(buildEventSubmitter()) : Optional.<EventSubmitter>absent();

  this.yarnConfiguration = yarnConfiguration;
  this.fs = fs;

  this.amrmClientAsync = closer.register(
      AMRMClientAsync.createAMRMClientAsync(1000, new AMRMClientCallbackHandler()));
  this.amrmClientAsync.init(this.yarnConfiguration);
  this.nmClientAsync = closer.register(NMClientAsync.createNMClientAsync(getNMClientCallbackHandler()));
  this.nmClientAsync.init(this.yarnConfiguration);

  this.initialContainers = config.getInt(GobblinYarnConfigurationKeys.INITIAL_CONTAINERS_KEY);
  this.requestedContainerMemoryMbs = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY);
  this.requestedContainerCores = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_CORES_KEY);
  this.containerHostAffinityEnabled = config.getBoolean(GobblinYarnConfigurationKeys.CONTAINER_HOST_AFFINITY_ENABLED);

  this.helixInstanceMaxRetries = config.getInt(GobblinYarnConfigurationKeys.HELIX_INSTANCE_MAX_RETRIES);
  this.helixInstanceTags = ConfigUtils.getString(config, GobblinClusterConfigurationKeys.HELIX_INSTANCE_TAGS_KEY, null);

  this.containerJvmArgs = config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY) ?
      Optional.of(config.getString(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY)) :
      Optional.<String>absent();

  this.containerLaunchExecutor = ScalingThreadPoolExecutor.newScalingThreadPool(5, Integer.MAX_VALUE, 0L,
      ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("ContainerLaunchExecutor")));

  this.tokens = getSecurityTokens();

  this.releasedContainerCache = CacheBuilder.newBuilder().expireAfterAccess(ConfigUtils.getInt(config,
      GobblinYarnConfigurationKeys.RELEASED_CONTAINERS_CACHE_EXPIRY_SECS,
      GobblinYarnConfigurationKeys.DEFAULT_RELEASED_CONTAINERS_CACHE_EXPIRY_SECS), TimeUnit.SECONDS).build();

  this.jvmMemoryXmxRatio = ConfigUtils.getDouble(this.config,
      GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY,
      GobblinYarnConfigurationKeys.DEFAULT_CONTAINER_JVM_MEMORY_XMX_RATIO);

  Preconditions.checkArgument(this.jvmMemoryXmxRatio >= 0 && this.jvmMemoryXmxRatio <= 1,
      GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY + " must be between 0 and 1 inclusive");

  this.jvmMemoryOverheadMbs = ConfigUtils.getInt(this.config,
      GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY,
      GobblinYarnConfigurationKeys.DEFAULT_CONTAINER_JVM_MEMORY_OVERHEAD_MBS);

  Preconditions.checkArgument(this.jvmMemoryOverheadMbs < this.requestedContainerMemoryMbs * this.jvmMemoryXmxRatio,
      GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY + " cannot be more than "
          + GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY + " * "
          + GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY);

  this.appViewAcl = ConfigUtils.getString(this.config, GobblinYarnConfigurationKeys.APP_VIEW_ACL,
      GobblinYarnConfigurationKeys.DEFAULT_APP_VIEW_ACL);
  this.containerTimezone = ConfigUtils.getString(this.config, GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_TIMEZONE,
      GobblinYarnConfigurationKeys.DEFAULT_GOBBLIN_YARN_CONTAINER_TIMEZONE);
}
 

@Override
protected NMClientAsync createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	return testingYarnNMClientAsync;
}
 

public NMClientAsyncImpl(CallbackHandler callbackHandler) {
  this(NMClientAsync.class.getName(), callbackHandler);
}
 

public NMClientAsyncImpl(CallbackHandler callbackHandler) {
  this(NMClientAsync.class.getName(), callbackHandler);
}
 

NMClientAsync getNMClientAsync() {
  return nmClientAsync;
}
 

@Test
public void testAmShutdown() throws IOException, YarnException {

  Assume.assumeTrue(
      "This test requires a YARN Resource Manager to connect to",
      Boolean.parseBoolean(System.getenv("REEF_TEST_YARN")));

  final YarnConfiguration yarnConfig = new YarnConfiguration();

  // Start YARN client and register the application

  final YarnClient yarnClient = YarnClient.createYarnClient();
  yarnClient.init(yarnConfig);
  yarnClient.start();

  final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class);
  containerContext.setCommands(Collections.<String>emptyList());
  containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap());
  containerContext.setEnvironment(Collections.<String, String>emptyMap());
  containerContext.setTokens(getTokens());

  final ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext();
  appContext.setApplicationName("REEF_Unmanaged_AM_Test");
  appContext.setAMContainerSpec(containerContext);
  appContext.setUnmanagedAM(true);
  appContext.setQueue("default");

  final ApplicationId applicationId = appContext.getApplicationId();
  LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId);

  yarnClient.submitApplication(appContext);

  LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId);

  addToken(yarnClient.getAMRMToken(applicationId));

  // Start the AM

  final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this);
  rmClient.init(yarnConfig);
  rmClient.start();

  final NMClientAsync nmClient = new NMClientAsyncImpl(this);
  nmClient.init(yarnConfig);
  nmClient.start();

  final RegisterApplicationMasterResponse registration =
      rmClient.registerApplicationMaster(NetUtils.getHostname(), -1, null);

  LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration);

  rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null);

  LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState());

  // Shutdown the AM

  rmClient.stop();
  nmClient.stop();

  // Get the final application report

  final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);
  final YarnApplicationState appState = appReport.getYarnApplicationState();
  final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus();

  LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] {
      applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus});

  Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState);
  Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus);

  // Shutdown YARN client

  yarnClient.stop();
}