Java Code Examples for org.apache.hadoop.yarn.client.api.AMRMClient#createAMRMClient()

The following examples show how to use org.apache.hadoop.yarn.client.api.AMRMClient#createAMRMClient() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestAMRMClient.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testWaitFor() throws InterruptedException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  CountDownSupplier countDownChecker = new CountDownSupplier();

  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
            .<ContainerRequest> createAMRMClient();
    amClient.init(new YarnConfiguration());
    amClient.start();
    amClient.waitFor(countDownChecker, 1000);
    assertEquals(3, countDownChecker.counter);
  } finally {
    if (amClient != null) {
      amClient.stop();
    }
  }
}
Example 2
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testWaitFor() throws InterruptedException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  CountDownSupplier countDownChecker = new CountDownSupplier();

  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
            .<ContainerRequest> createAMRMClient();
    amClient.init(new YarnConfiguration());
    amClient.start();
    amClient.waitFor(countDownChecker, 1000);
    assertEquals(3, countDownChecker.counter);
  } finally {
    if (amClient != null) {
      amClient.stop();
    }
  }
}
Example 3
Source File: TestAMRMClient.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test (timeout=60000)
public void testAMRMClient() throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.<ContainerRequest>createAMRMClient();

    //setting an instance NMTokenCache
    amClient.setNMTokenCache(new NMTokenCache());
    //asserting we are not using the singleton instance cache
    Assert.assertNotSame(NMTokenCache.getSingleton(), 
        amClient.getNMTokenCache());

    amClient.init(conf);
    amClient.start();

    amClient.registerApplicationMaster("Host", 10000, "");

    testAllocation((AMRMClientImpl<ContainerRequest>)amClient);

    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
Example 4
Source File: Hadoop21YarnAMClient.java    From twill with Apache License 2.0 5 votes vote down vote up 
public Hadoop21YarnAMClient(Configuration conf) {
  super(ApplicationConstants.Environment.CONTAINER_ID.name());

  this.amrmClient = AMRMClient.createAMRMClient();
  this.amrmClient.init(conf);
  this.nmClient = new Hadoop21YarnNMClient(conf);
}
Example 5
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test (timeout=60000)
public void testAMRMClient() throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.<ContainerRequest>createAMRMClient();

    //setting an instance NMTokenCache
    amClient.setNMTokenCache(new NMTokenCache());
    //asserting we are not using the singleton instance cache
    Assert.assertNotSame(NMTokenCache.getSingleton(), 
        amClient.getNMTokenCache());

    amClient.init(conf);
    amClient.start();

    amClient.registerApplicationMaster("Host", 10000, "");

    testAllocation((AMRMClientImpl<ContainerRequest>)amClient);

    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
Example 6
Source File: YarnManager.java    From Scribengin with GNU Affero General Public License v3.0 5 votes vote down vote up 
@Inject
public void onInit(VMConfig vmConfig) throws Exception {
  logger.info("Start init(VMConfig vmConfig)");
  this.vmConfig = vmConfig;
  try {
    this.yarnConfig = vmConfig.getHadoopProperties();
    conf = new YarnConfiguration() ;
    vmConfig.overrideHadoopConfiguration(conf);
    
    amrmClient = AMRMClient.createAMRMClient();
    amrmClientAsync = AMRMClientAsync.createAMRMClientAsync(amrmClient, 1000, new AMRMCallbackHandler());
    amrmClientAsync.init(conf);
    amrmClientAsync.start();
    
    nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();
    // Register with RM
    String appHostName = InetAddress.getLocalHost().getHostAddress()  ;

    RegisterApplicationMasterResponse registerResponse = amrmClientAsync.registerApplicationMaster(appHostName, 0, "");
    System.out.println("amrmClientAsync.registerApplicationMaster");
  } catch(Throwable t) {
    logger.error("Error: " , t);
    t.printStackTrace();
  }
  logger.info("Finish init(VMConfig vmConfig)");
}
Example 7
Source File: StreamingAppMasterService.java    From Bats with Apache License 2.0 4 votes vote down vote up 
@Override
protected void serviceInit(Configuration conf) throws Exception
{
  LOG.info("Application master" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId());

  FileInputStream fis = new FileInputStream("./" + LogicalPlan.SER_FILE_NAME);
  try {
    this.dag = LogicalPlan.read(fis);
  } finally {
    fis.close();
  }
  // "debug" simply dumps all data using LOG.info
  if (dag.isDebug()) {
    dumpOutDebugInfo();
  }
  dag.setAttribute(LogicalPlan.APPLICATION_ATTEMPT_ID, appAttemptID.getAttemptId());
  FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), conf);
  this.dnmgr = StreamingContainerManager.getInstance(recoveryHandler, dag, true);
  dag = this.dnmgr.getLogicalPlan();
  this.appContext = new ClusterAppContextImpl(dag.getAttributes());

  Map<Class<?>, Class<? extends StringCodec<?>>> codecs = dag.getAttributes().get(DAG.STRING_CODECS);
  StringCodecs.loadConverters(codecs);

  LOG.info("Starting application with {} operators in {} containers", dnmgr.getPhysicalPlan().getAllOperators().size(), dnmgr.getPhysicalPlan().getContainers().size());

  // Setup security configuration such as that for web security
  SecurityUtils.init(conf, dag.getValue(LogicalPlan.STRAM_HTTP_AUTHENTICATION));

  if (UserGroupInformation.isSecurityEnabled()) {
    // TODO :- Need to perform token renewal
    delegationTokenManager = new StramDelegationTokenManager(DELEGATION_KEY_UPDATE_INTERVAL, DELEGATION_TOKEN_MAX_LIFETIME, DELEGATION_TOKEN_RENEW_INTERVAL, DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL);
  }
  this.nmClient = new NMClientAsyncImpl(new NMCallbackHandler());
  addService(nmClient);
  this.amRmClient = AMRMClient.createAMRMClient();
  addService(amRmClient);

  // start RPC server
  int rpcListenerCount = dag.getValue(DAGContext.HEARTBEAT_LISTENER_THREAD_COUNT);
  this.heartbeatListener = new StreamingContainerParent(this.getClass().getName(), dnmgr, delegationTokenManager, rpcListenerCount);
  addService(heartbeatListener);

  AutoMetric.Transport appDataPushTransport = dag.getValue(LogicalPlan.METRICS_TRANSPORT);
  if (appDataPushTransport != null) {
    this.appDataPushAgent = new AppDataPushAgent(dnmgr, appContext);
    addService(this.appDataPushAgent);
  }
  initApexPluginDispatcher();

  // Initialize all services added above
  super.serviceInit(conf);
}
Example 8
Source File: TestNMClient.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}
Example 9
Source File: TestAMRMClient.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test (timeout=60000)
public void testAllocationWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    ContainerRequest storedContainer1 = 
        new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer1);
    assertEquals(3, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    List<String> localNodeBlacklist = new ArrayList<String>();
    localNodeBlacklist.add(node);
    
    // put node in black list, so no container assignment
    amClient.updateBlacklist(localNodeBlacklist, null);

    int allocatedContainerCount = getAllocatedContainersNumber(amClient,
      DEFAULT_ITERATION);
    // the only node is in blacklist, so no allocation
    assertEquals(0, allocatedContainerCount);

    // Remove node from blacklist, so get assigned with 2
    amClient.updateBlacklist(null, localNodeBlacklist);
    ContainerRequest storedContainer2 = 
            new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer2);
    allocatedContainerCount = getAllocatedContainersNumber(amClient,
        DEFAULT_ITERATION);
    assertEquals(2, allocatedContainerCount);
    
    // Test in case exception in allocate(), blacklist is kept
    assertTrue(amClient.blacklistAdditions.isEmpty());
    assertTrue(amClient.blacklistRemovals.isEmpty());
    
    // create a invalid ContainerRequest - memory value is minus
    ContainerRequest invalidContainerRequest = 
        new ContainerRequest(Resource.newInstance(-1024, 1, 1),
            nodes, racks, priority);
    amClient.addContainerRequest(invalidContainerRequest);
    amClient.updateBlacklist(localNodeBlacklist, null);
    try {
      // allocate() should complain as ContainerRequest is invalid.
      amClient.allocate(0.1f);
      fail("there should be an exception here.");
    } catch (Exception e) {
      assertEquals(1, amClient.blacklistAdditions.size());
    }
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
Example 10
Source File: TestAMRMClient.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test (timeout=60000)
public void testAMRMClientWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    String[] nodes = {"node1", "node2", "node3"};
    
    // Add nodes[0] and nodes[1]
    List<String> nodeList01 = new ArrayList<String>();
    nodeList01.add(nodes[0]);
    nodeList01.add(nodes[1]);
    amClient.updateBlacklist(nodeList01, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[0] again, verify it is not added duplicated.
    List<String> nodeList02 = new ArrayList<String>();
    nodeList02.add(nodes[0]);
    nodeList02.add(nodes[2]);
    amClient.updateBlacklist(nodeList02, null);
    assertEquals(3, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[1] and nodes[2] to removal list, 
    // Verify addition list remove these two nodes.
    List<String> nodeList12 = new ArrayList<String>();
    nodeList12.add(nodes[1]);
    nodeList12.add(nodes[2]);
    amClient.updateBlacklist(null, nodeList12);
    assertEquals(1, amClient.blacklistAdditions.size());
    assertEquals(2, amClient.blacklistRemovals.size());
    
    // Add nodes[1] again to addition list, 
    // Verify removal list will remove this node.
    List<String> nodeList1 = new ArrayList<String>();
    nodeList1.add(nodes[1]);
    amClient.updateBlacklist(nodeList1, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(1, amClient.blacklistRemovals.size());
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
Example 11
Source File: TestNMClient.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}
Example 12
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test (timeout=60000)
public void testAllocationWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    ContainerRequest storedContainer1 = 
        new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer1);
    assertEquals(3, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    List<String> localNodeBlacklist = new ArrayList<String>();
    localNodeBlacklist.add(node);
    
    // put node in black list, so no container assignment
    amClient.updateBlacklist(localNodeBlacklist, null);

    int allocatedContainerCount = getAllocatedContainersNumber(amClient,
      DEFAULT_ITERATION);
    // the only node is in blacklist, so no allocation
    assertEquals(0, allocatedContainerCount);

    // Remove node from blacklist, so get assigned with 2
    amClient.updateBlacklist(null, localNodeBlacklist);
    ContainerRequest storedContainer2 = 
            new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer2);
    allocatedContainerCount = getAllocatedContainersNumber(amClient,
        DEFAULT_ITERATION);
    assertEquals(2, allocatedContainerCount);
    
    // Test in case exception in allocate(), blacklist is kept
    assertTrue(amClient.blacklistAdditions.isEmpty());
    assertTrue(amClient.blacklistRemovals.isEmpty());
    
    // create a invalid ContainerRequest - memory value is minus
    ContainerRequest invalidContainerRequest = 
        new ContainerRequest(Resource.newInstance(-1024, 1),
            nodes, racks, priority);
    amClient.addContainerRequest(invalidContainerRequest);
    amClient.updateBlacklist(localNodeBlacklist, null);
    try {
      // allocate() should complain as ContainerRequest is invalid.
      amClient.allocate(0.1f);
      fail("there should be an exception here.");
    } catch (Exception e) {
      assertEquals(1, amClient.blacklistAdditions.size());
    }
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
Example 13
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test (timeout=60000)
public void testAMRMClientWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    String[] nodes = {"node1", "node2", "node3"};
    
    // Add nodes[0] and nodes[1]
    List<String> nodeList01 = new ArrayList<String>();
    nodeList01.add(nodes[0]);
    nodeList01.add(nodes[1]);
    amClient.updateBlacklist(nodeList01, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[0] again, verify it is not added duplicated.
    List<String> nodeList02 = new ArrayList<String>();
    nodeList02.add(nodes[0]);
    nodeList02.add(nodes[2]);
    amClient.updateBlacklist(nodeList02, null);
    assertEquals(3, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[1] and nodes[2] to removal list, 
    // Verify addition list remove these two nodes.
    List<String> nodeList12 = new ArrayList<String>();
    nodeList12.add(nodes[1]);
    nodeList12.add(nodes[2]);
    amClient.updateBlacklist(null, nodeList12);
    assertEquals(1, amClient.blacklistAdditions.size());
    assertEquals(2, amClient.blacklistRemovals.size());
    
    // Add nodes[1] again to addition list, 
    // Verify removal list will remove this node.
    List<String> nodeList1 = new ArrayList<String>();
    nodeList1.add(nodes[1]);
    amClient.updateBlacklist(nodeList1, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(1, amClient.blacklistRemovals.size());
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
Example 14
Source File: StreamingAppMasterService.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up 
@Override
protected void serviceInit(Configuration conf) throws Exception
{
  LOG.info("Application master" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId());

  FileInputStream fis = new FileInputStream("./" + LogicalPlan.SER_FILE_NAME);
  try {
    this.dag = LogicalPlan.read(fis);
  } finally {
    fis.close();
  }
  // "debug" simply dumps all data using LOG.info
  if (dag.isDebug()) {
    dumpOutDebugInfo();
  }
  dag.setAttribute(LogicalPlan.APPLICATION_ATTEMPT_ID, appAttemptID.getAttemptId());
  FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), conf);
  this.dnmgr = StreamingContainerManager.getInstance(recoveryHandler, dag, true);
  dag = this.dnmgr.getLogicalPlan();
  this.appContext = new ClusterAppContextImpl(dag.getAttributes());

  Map<Class<?>, Class<? extends StringCodec<?>>> codecs = dag.getAttributes().get(DAG.STRING_CODECS);
  StringCodecs.loadConverters(codecs);

  LOG.info("Starting application with {} operators in {} containers", dnmgr.getPhysicalPlan().getAllOperators().size(), dnmgr.getPhysicalPlan().getContainers().size());

  // Setup security configuration such as that for web security
  SecurityUtils.init(conf, dag.getValue(LogicalPlan.STRAM_HTTP_AUTHENTICATION));

  if (UserGroupInformation.isSecurityEnabled()) {
    // TODO :- Need to perform token renewal
    delegationTokenManager = new StramDelegationTokenManager(DELEGATION_KEY_UPDATE_INTERVAL, DELEGATION_TOKEN_MAX_LIFETIME, DELEGATION_TOKEN_RENEW_INTERVAL, DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL);
  }
  this.nmClient = new NMClientAsyncImpl(new NMCallbackHandler());
  addService(nmClient);
  this.amRmClient = AMRMClient.createAMRMClient();
  addService(amRmClient);

  // start RPC server
  int rpcListenerCount = dag.getValue(DAGContext.HEARTBEAT_LISTENER_THREAD_COUNT);
  this.heartbeatListener = new StreamingContainerParent(this.getClass().getName(), dnmgr, delegationTokenManager, rpcListenerCount);
  addService(heartbeatListener);

  AutoMetric.Transport appDataPushTransport = dag.getValue(LogicalPlan.METRICS_TRANSPORT);
  if (appDataPushTransport != null) {
    this.appDataPushAgent = new AppDataPushAgent(dnmgr, appContext);
    addService(this.appDataPushAgent);
  }
  initApexPluginDispatcher();

  // Initialize all services added above
  super.serviceInit(conf);
}