Java Code Examples for org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest#newInstance()

The following examples show how to use org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest#newInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestAllocateRequest.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAllcoateRequestWithIncrease() {
  List<ContainerResourceIncreaseRequest> incRequests =
      new ArrayList<ContainerResourceIncreaseRequest>();
  for (int i = 0; i < 3; i++) {
    incRequests.add(ContainerResourceIncreaseRequest.newInstance(null,
        Resource.newInstance(0, i)));
  }
  AllocateRequest r =
      AllocateRequest.newInstance(123, 0f, null, null, null, incRequests);

  // serde
  AllocateRequestProto p = ((AllocateRequestPBImpl) r).getProto();
  r = new AllocateRequestPBImpl(p);

  // check value
  Assert.assertEquals(123, r.getResponseId());
  Assert.assertEquals(incRequests.size(), r.getIncreaseRequests().size());

  for (int i = 0; i < incRequests.size(); i++) {
    Assert.assertEquals(r.getIncreaseRequests().get(i).getCapability()
        .getVirtualCores(), incRequests.get(i).getCapability()
        .getVirtualCores());
  }
}
Example 2
Source File: TestAllocateRequest.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAllcoateRequestWithIncrease() {
  List<ContainerResourceIncreaseRequest> incRequests =
      new ArrayList<ContainerResourceIncreaseRequest>();
  for (int i = 0; i < 3; i++) {
    incRequests.add(ContainerResourceIncreaseRequest.newInstance(null,
        Resource.newInstance(0, i)));
  }
  AllocateRequest r =
      AllocateRequest.newInstance(123, 0f, null, null, null, incRequests);

  // serde
  AllocateRequestProto p = ((AllocateRequestPBImpl) r).getProto();
  r = new AllocateRequestPBImpl(p);

  // check value
  Assert.assertEquals(123, r.getResponseId());
  Assert.assertEquals(incRequests.size(), r.getIncreaseRequests().size());

  for (int i = 0; i < incRequests.size(); i++) {
    Assert.assertEquals(r.getIncreaseRequests().get(i).getCapability()
        .getVirtualCores(), incRequests.get(i).getCapability()
        .getVirtualCores());
  }
}
Example 3
Source File: TestAllocateRequest.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAllcoateRequestWithoutIncrease() {
  AllocateRequest r =
      AllocateRequest.newInstance(123, 0f, null, null, null, null);

  // serde
  AllocateRequestProto p = ((AllocateRequestPBImpl) r).getProto();
  r = new AllocateRequestPBImpl(p);

  // check value
  Assert.assertEquals(123, r.getResponseId());
  Assert.assertEquals(0, r.getIncreaseRequests().size());
}
Example 4
Source File: MockAM.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public AllocateResponse allocate(
    List<ResourceRequest> resourceRequest, List<ContainerId> releases)
    throws Exception {
  final AllocateRequest req =
      AllocateRequest.newInstance(0, 0F, resourceRequest,
        releases, null);
  return allocate(req);
}
Example 5
Source File: TestApplicationMasterServiceProtocolOnHA.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 15000)
public void testAllocateOnHA() throws YarnException, IOException {
  AllocateRequest request = AllocateRequest.newInstance(0, 50f,
      new ArrayList<ResourceRequest>(),
      new ArrayList<ContainerId>(),
      ResourceBlacklistRequest.newInstance(new ArrayList<String>(),
          new ArrayList<String>()));
  AllocateResponse response = amClient.allocate(request);
  Assert.assertEquals(response, this.cluster.createFakeAllocateResponse());
}
Example 6
Source File: TestAllocateRequest.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAllcoateRequestWithoutIncrease() {
  AllocateRequest r =
      AllocateRequest.newInstance(123, 0f, null, null, null, null);

  // serde
  AllocateRequestProto p = ((AllocateRequestPBImpl) r).getProto();
  r = new AllocateRequestPBImpl(p);

  // check value
  Assert.assertEquals(123, r.getResponseId());
  Assert.assertEquals(0, r.getIncreaseRequests().size());
}
Example 7
Source File: MockAM.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public AllocateResponse allocate(
    List<ResourceRequest> resourceRequest, List<ContainerId> releases)
    throws Exception {
  final AllocateRequest req =
      AllocateRequest.newInstance(0, 0F, resourceRequest,
        releases, null);
  return allocate(req);
}
Example 8
Source File: TestApplicationMasterServiceProtocolOnHA.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 15000)
public void testAllocateOnHA() throws YarnException, IOException {
  AllocateRequest request = AllocateRequest.newInstance(0, 50f,
      new ArrayList<ResourceRequest>(),
      new ArrayList<ContainerId>(),
      ResourceBlacklistRequest.newInstance(new ArrayList<String>(),
          new ArrayList<String>()));
  AllocateResponse response = amClient.allocate(request);
  Assert.assertEquals(response, this.cluster.createFakeAllocateResponse());
}
Example 9
Source File: TestAMRMRPCResponseId.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testARRMResponseId() throws Exception {

  MockNM nm1 = rm.registerNode("h1:1234", 5000);

  RMApp app = rm.submitApp(2000);

  // Trigger the scheduling so the AM gets 'launched'
  nm1.nodeHeartbeat(true);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());

  am.registerAppAttempt();
  
  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(0, 0F, null, null, null);

  AllocateResponse response =
      allocate(attempt.getAppAttemptId(), allocateRequest);
  Assert.assertEquals(1, response.getResponseId());
  Assert.assertTrue(response.getAMCommand() == null);
  allocateRequest =
      AllocateRequest.newInstance(response.getResponseId(), 0F, null, null,
        null);
  
  response = allocate(attempt.getAppAttemptId(), allocateRequest);
  Assert.assertEquals(2, response.getResponseId());
  /* try resending */
  response = allocate(attempt.getAppAttemptId(), allocateRequest);
  Assert.assertEquals(2, response.getResponseId());
  
  /** try sending old request again **/
  allocateRequest = AllocateRequest.newInstance(0, 0F, null, null, null);

  try {
    allocate(attempt.getAppAttemptId(), allocateRequest);
    Assert.fail();
  } catch (Exception e) {
    Assert.assertTrue(e.getCause() instanceof InvalidApplicationMasterRequestException);
  }
}
Example 10
Source File: TestApplicationCleanup.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 60000)
public void testProcessingNMContainerStatusesOnNMRestart() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  // 1. Start the cluster-RM,NM,Submit app with 1024MB,Launch & register AM
  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  int nmMemory = 8192;
  int amMemory = 1024;
  int containerMemory = 2048;
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", nmMemory, rm1.getResourceTrackerService());
  nm1.registerNode();

  RMApp app0 = rm1.submitApp(amMemory);
  MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);

  // 2. AM sends ResourceRequest for 1 container with memory 2048MB.
  int noOfContainers = 1;
  List<Container> allocateContainers =
      am0.allocateAndWaitForContainers(noOfContainers, containerMemory, nm1);

  // 3. Verify for number of container allocated by RM
  Assert.assertEquals(noOfContainers, allocateContainers.size());
  Container container = allocateContainers.get(0);

  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.RUNNING);
  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), container.getId()
      .getContainerId(), ContainerState.RUNNING);

  rm1.waitForState(app0.getApplicationId(), RMAppState.RUNNING);

  // 4. Verify Memory Usage by cluster, it should be 3072. AM memory +
  // requested memory. 1024 + 2048=3072
  ResourceScheduler rs = rm1.getRMContext().getScheduler();
  int allocatedMB = rs.getRootQueueMetrics().getAllocatedMB();
  Assert.assertEquals(amMemory + containerMemory, allocatedMB);

  // 5. Re-register NM by sending completed container status
  List<NMContainerStatus> nMContainerStatusForApp =
      createNMContainerStatusForApp(am0);
  nm1.registerNode(nMContainerStatusForApp,
      Arrays.asList(app0.getApplicationId()));

  waitForClusterMemory(nm1, rs, amMemory);

  // 6. Verify for Memory Used, it should be 1024
  Assert.assertEquals(amMemory, rs.getRootQueueMetrics().getAllocatedMB());

  // 7. Send AM heatbeat to RM. Allocated response should contain completed
  // container
  AllocateRequest req =
      AllocateRequest.newInstance(0, 0F, new ArrayList<ResourceRequest>(),
          new ArrayList<ContainerId>(), null);
  AllocateResponse allocate = am0.allocate(req);
  List<ContainerStatus> completedContainersStatuses =
      allocate.getCompletedContainersStatuses();
  Assert.assertEquals(noOfContainers, completedContainersStatuses.size());

  // Application clean up should happen Cluster memory used is 0
  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  waitForClusterMemory(nm1, rs, 0);

  rm1.stop();
}
Example 11
Source File: TestSchedulerUtils.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testValidateResourceBlacklistRequest() throws Exception {

  MyContainerManager containerManager = new MyContainerManager();
  final MockRMWithAMS rm =
      new MockRMWithAMS(new YarnConfiguration(), containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>(2);
  acls.put(ApplicationAccessType.VIEW_APP, "*");
  RMApp app = rm.submitApp(1024, "appname", "appuser", acls);

  nm1.nodeHeartbeat(true);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  // Create a client to the RM.
  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);

  UserGroupInformation currentUser = 
      UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
  Credentials credentials = containerManager.getContainerCredentials();
  final InetSocketAddress rmBindAddress =
      rm.getApplicationMasterService().getBindAddress();
  Token<? extends TokenIdentifier> amRMToken =
      MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
        credentials.getAllTokens());
  currentUser.addToken(amRMToken);
  ApplicationMasterProtocol client =
      currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(
            ApplicationMasterProtocol.class, rmBindAddress, conf);
        }
      });

  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  client.registerApplicationMaster(request);

  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(
          Collections.singletonList(ResourceRequest.ANY), null);

  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(0, 0.0f, null, null, blacklistRequest);
  boolean error = false;
  try {
    client.allocate(allocateRequest);
  } catch (InvalidResourceBlacklistRequestException e) {
    error = true;
  }

  rm.stop();
  
  Assert.assertTrue(
      "Didn't not catch InvalidResourceBlacklistRequestException", error);
}
Example 12
Source File: RMContainerRequestor.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
protected AllocateResponse makeRemoteRequest() throws YarnException,
    IOException {
  applyRequestLimits();
  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(new ArrayList<String>(blacklistAdditions),
          new ArrayList<String>(blacklistRemovals));
  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(lastResponseID,
        super.getApplicationProgress(), new ArrayList<ResourceRequest>(ask),
        new ArrayList<ContainerId>(release), blacklistRequest);
  AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
  lastResponseID = allocateResponse.getResponseId();
  availableResources = allocateResponse.getAvailableResources();
  lastClusterNmCount = clusterNmCount;
  clusterNmCount = allocateResponse.getNumClusterNodes();
  int numCompletedContainers =
      allocateResponse.getCompletedContainersStatuses().size();

  if (ask.size() > 0 || release.size() > 0) {
    LOG.info("getResources() for " + applicationId + ":" + " ask="
        + ask.size() + " release= " + release.size() + " newContainers="
        + allocateResponse.getAllocatedContainers().size()
        + " finishedContainers=" + numCompletedContainers
        + " resourcelimit=" + availableResources + " knownNMs="
        + clusterNmCount);
  }

  ask.clear();
  release.clear();

  if (numCompletedContainers > 0) {
    // re-send limited requests when a container completes to trigger asking
    // for more containers
    requestLimitsToUpdate.addAll(requestLimits.keySet());
  }

  if (blacklistAdditions.size() > 0 || blacklistRemovals.size() > 0) {
    LOG.info("Update the blacklist for " + applicationId +
        ": blacklistAdditions=" + blacklistAdditions.size() +
        " blacklistRemovals=" +  blacklistRemovals.size());
  }
  blacklistAdditions.clear();
  blacklistRemovals.clear();
  return allocateResponse;
}
Example 13
Source File: TestAMRMRPCResponseId.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testARRMResponseId() throws Exception {

  MockNM nm1 = rm.registerNode("h1:1234", 5000);

  RMApp app = rm.submitApp(2000);

  // Trigger the scheduling so the AM gets 'launched'
  nm1.nodeHeartbeat(true);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());

  am.registerAppAttempt();
  
  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(0, 0F, null, null, null);

  AllocateResponse response =
      allocate(attempt.getAppAttemptId(), allocateRequest);
  Assert.assertEquals(1, response.getResponseId());
  Assert.assertTrue(response.getAMCommand() == null);
  allocateRequest =
      AllocateRequest.newInstance(response.getResponseId(), 0F, null, null,
        null);
  
  response = allocate(attempt.getAppAttemptId(), allocateRequest);
  Assert.assertEquals(2, response.getResponseId());
  /* try resending */
  response = allocate(attempt.getAppAttemptId(), allocateRequest);
  Assert.assertEquals(2, response.getResponseId());
  
  /** try sending old request again **/
  allocateRequest = AllocateRequest.newInstance(0, 0F, null, null, null);

  try {
    allocate(attempt.getAppAttemptId(), allocateRequest);
    Assert.fail();
  } catch (Exception e) {
    Assert.assertTrue(e.getCause() instanceof InvalidApplicationMasterRequestException);
  }
}
Example 14
Source File: TestApplicationCleanup.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 60000)
public void testProcessingNMContainerStatusesOnNMRestart() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  // 1. Start the cluster-RM,NM,Submit app with 1024MB,Launch & register AM
  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  int nmMemory = 8192;
  int amMemory = 1024;
  int containerMemory = 2048;
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", nmMemory, rm1.getResourceTrackerService());
  nm1.registerNode();

  RMApp app0 = rm1.submitApp(amMemory);
  MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);

  // 2. AM sends ResourceRequest for 1 container with memory 2048MB.
  int noOfContainers = 1;
  List<Container> allocateContainers =
      am0.allocateAndWaitForContainers(noOfContainers, containerMemory, nm1);

  // 3. Verify for number of container allocated by RM
  Assert.assertEquals(noOfContainers, allocateContainers.size());
  Container container = allocateContainers.get(0);

  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.RUNNING);
  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), container.getId()
      .getContainerId(), ContainerState.RUNNING);

  rm1.waitForState(app0.getApplicationId(), RMAppState.RUNNING);

  // 4. Verify Memory Usage by cluster, it should be 3072. AM memory +
  // requested memory. 1024 + 2048=3072
  ResourceScheduler rs = rm1.getRMContext().getScheduler();
  int allocatedMB = rs.getRootQueueMetrics().getAllocatedMB();
  Assert.assertEquals(amMemory + containerMemory, allocatedMB);

  // 5. Re-register NM by sending completed container status
  List<NMContainerStatus> nMContainerStatusForApp =
      createNMContainerStatusForApp(am0);
  nm1.registerNode(nMContainerStatusForApp,
      Arrays.asList(app0.getApplicationId()));

  waitForClusterMemory(nm1, rs, amMemory);

  // 6. Verify for Memory Used, it should be 1024
  Assert.assertEquals(amMemory, rs.getRootQueueMetrics().getAllocatedMB());

  // 7. Send AM heatbeat to RM. Allocated response should contain completed
  // container
  AllocateRequest req =
      AllocateRequest.newInstance(0, 0F, new ArrayList<ResourceRequest>(),
          new ArrayList<ContainerId>(), null);
  AllocateResponse allocate = am0.allocate(req);
  List<ContainerStatus> completedContainersStatuses =
      allocate.getCompletedContainersStatuses();
  Assert.assertEquals(noOfContainers, completedContainersStatuses.size());

  // Application clean up should happen Cluster memory used is 0
  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  waitForClusterMemory(nm1, rs, 0);

  rm1.stop();
}
Example 15
Source File: TestSchedulerUtils.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testValidateResourceBlacklistRequest() throws Exception {

  MyContainerManager containerManager = new MyContainerManager();
  final MockRMWithAMS rm =
      new MockRMWithAMS(new YarnConfiguration(), containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>(2);
  acls.put(ApplicationAccessType.VIEW_APP, "*");
  RMApp app = rm.submitApp(1024, "appname", "appuser", acls);

  nm1.nodeHeartbeat(true);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  // Create a client to the RM.
  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);

  UserGroupInformation currentUser = 
      UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
  Credentials credentials = containerManager.getContainerCredentials();
  final InetSocketAddress rmBindAddress =
      rm.getApplicationMasterService().getBindAddress();
  Token<? extends TokenIdentifier> amRMToken =
      MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
        credentials.getAllTokens());
  currentUser.addToken(amRMToken);
  ApplicationMasterProtocol client =
      currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(
            ApplicationMasterProtocol.class, rmBindAddress, conf);
        }
      });

  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  client.registerApplicationMaster(request);

  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(
          Collections.singletonList(ResourceRequest.ANY), null);

  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(0, 0.0f, null, null, blacklistRequest);
  boolean error = false;
  try {
    client.allocate(allocateRequest);
  } catch (InvalidResourceBlacklistRequestException e) {
    error = true;
  }

  rm.stop();
  
  Assert.assertTrue(
      "Didn't not catch InvalidResourceBlacklistRequestException", error);
}
Example 16
Source File: RMContainerRequestor.java    From big-c with Apache License 2.0 4 votes vote down vote up 
protected AllocateResponse makeRemoteRequest() throws YarnException,
    IOException {
  applyRequestLimits();
  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(new ArrayList<String>(blacklistAdditions),
          new ArrayList<String>(blacklistRemovals));
  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(lastResponseID,
        super.getApplicationProgress(), new ArrayList<ResourceRequest>(ask),
        new ArrayList<ContainerId>(release), blacklistRequest);
  AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
  lastResponseID = allocateResponse.getResponseId();
  availableResources = allocateResponse.getAvailableResources();
  lastClusterNmCount = clusterNmCount;
  clusterNmCount = allocateResponse.getNumClusterNodes();
  int numCompletedContainers =
      allocateResponse.getCompletedContainersStatuses().size();

  if (ask.size() > 0 || release.size() > 0) {
    LOG.info("getResources() for " + applicationId + ":" + " ask="
        + ask.size() + " release= " + release.size() + " newContainers="
        + allocateResponse.getAllocatedContainers().size()
        + " finishedContainers=" + numCompletedContainers
        + " resourcelimit=" + availableResources + " knownNMs="
        + clusterNmCount);
  }

  ask.clear();
  release.clear();

  if (numCompletedContainers > 0) {
    // re-send limited requests when a container completes to trigger asking
    // for more containers
    requestLimitsToUpdate.addAll(requestLimits.keySet());
  }

  if (blacklistAdditions.size() > 0 || blacklistRemovals.size() > 0) {
    LOG.info("Update the blacklist for " + applicationId +
        ": blacklistAdditions=" + blacklistAdditions.size() +
        " blacklistRemovals=" +  blacklistRemovals.size());
  }
  blacklistAdditions.clear();
  blacklistRemovals.clear();
  return allocateResponse;
}