Java Code Examples for org.apache.hadoop.yarn.api.records.ResourceRequest#setNumContainers()

The following examples show how to use org.apache.hadoop.yarn.api.records.ResourceRequest#setNumContainers() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AppSchedulingInfo.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
synchronized private void decrementOutstanding(
    ResourceRequest offSwitchRequest) {
  int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1;

  // Do not remove ANY
  offSwitchRequest.setNumContainers(numOffSwitchContainers);
  
  // Do we have any outstanding requests?
  // If there is nothing, we need to deactivate this application
  if (numOffSwitchContainers == 0) {
    checkForDeactivation();
  }
  
  queue.decPendingResource(offSwitchRequest.getNodeLabelExpression(),
      offSwitchRequest.getCapability());
}
Example 2
Source File: MockAM.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public ResourceRequest createResourceReq(String resource, int memory, int priority,
    int containers, String labelExpression) throws Exception {
  ResourceRequest req = Records.newRecord(ResourceRequest.class);
  req.setResourceName(resource);
  req.setNumContainers(containers);
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(priority);
  req.setPriority(pri);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(memory);
  req.setCapability(capability);
  if (labelExpression != null) {
   req.setNodeLabelExpression(labelExpression); 
  }
  return req;
}
Example 3
Source File: TestUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}
Example 4
Source File: AppSchedulingInfo.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void decResourceRequest(String resourceName, Priority priority,
    ResourceRequest request) {
  request.setNumContainers(request.getNumContainers() - 1);
  if (request.getNumContainers() == 0) {
    requests.get(priority).remove(resourceName);
  }
}
Example 5
Source File: FairSchedulerTestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected ResourceRequest createResourceRequest(
    int memory, int vcores, String host, int priority, int numContainers,
    boolean relaxLocality) {
  ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class);
  request.setCapability(BuilderUtils.newResource(memory, vcores));
  request.setResourceName(host);
  request.setNumContainers(numContainers);
  Priority prio = recordFactory.newRecordInstance(Priority.class);
  prio.setPriority(priority);
  request.setPriority(prio);
  request.setRelaxLocality(relaxLocality);
  return request;
}
Example 6
Source File: BuilderUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static ResourceRequest newResourceRequest(Priority priority,
    String hostName, Resource capability, int numContainers) {
  ResourceRequest request = recordFactory
      .newRecordInstance(ResourceRequest.class);
  request.setPriority(priority);
  request.setResourceName(hostName);
  request.setCapability(capability);
  request.setNumContainers(numContainers);
  return request;
}
Example 7
Source File: AMSimulator.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
protected ResourceRequest createResourceRequest(
        Resource resource, String host, int priority, int numContainers) {
  ResourceRequest request = recordFactory
      .newRecordInstance(ResourceRequest.class);
  request.setCapability(resource);
  request.setResourceName(host);
  request.setNumContainers(numContainers);
  Priority prio = recordFactory.newRecordInstance(Priority.class);
  prio.setPriority(priority);
  request.setPriority(prio);
  return request;
}
Example 8
Source File: RMContainerRequestor.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void addResourceRequest(Priority priority, String resourceName,
    Resource capability, String nodeLabelExpression) {
  Map<String, Map<Resource, ResourceRequest>> remoteRequests =
    this.remoteRequestsTable.get(priority);
  if (remoteRequests == null) {
    remoteRequests = new HashMap<String, Map<Resource, ResourceRequest>>();
    this.remoteRequestsTable.put(priority, remoteRequests);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Added priority=" + priority);
    }
  }
  Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
  if (reqMap == null) {
    reqMap = new HashMap<Resource, ResourceRequest>();
    remoteRequests.put(resourceName, reqMap);
  }
  ResourceRequest remoteRequest = reqMap.get(capability);
  if (remoteRequest == null) {
    remoteRequest = recordFactory.newRecordInstance(ResourceRequest.class);
    remoteRequest.setPriority(priority);
    remoteRequest.setResourceName(resourceName);
    remoteRequest.setCapability(capability);
    remoteRequest.setNumContainers(0);
    remoteRequest.setNodeLabelExpression(nodeLabelExpression);
    reqMap.put(capability, remoteRequest);
  }
  remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1);

  // Note this down for next interaction with ResourceManager
  addResourceRequestToAsk(remoteRequest);
  if (LOG.isDebugEnabled()) {
    LOG.debug("addResourceRequest:" + " applicationId="
        + applicationId.getId() + " priority=" + priority.getPriority()
        + " resourceName=" + resourceName + " numContainers="
        + remoteRequest.getNumContainers() + " #asks=" + ask.size());
  }
}
Example 9
Source File: TestUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}
Example 10
Source File: AppSchedulingInfo.java    From big-c with Apache License 2.0 5 votes vote down vote up 
synchronized private void decrementOutstanding(
    ResourceRequest offSwitchRequest) {
  int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1;

  // Do not remove ANY
  offSwitchRequest.setNumContainers(numOffSwitchContainers);
  
  // Do we have any outstanding requests?
  // If there is nothing, we need to deactivate this application
  if (numOffSwitchContainers == 0) {
    checkForDeactivation();
  }
}
Example 11
Source File: AMSimulator.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected ResourceRequest createResourceRequest(
        Resource resource, String host, int priority, int numContainers) {
  ResourceRequest request = recordFactory
      .newRecordInstance(ResourceRequest.class);
  request.setCapability(resource);
  request.setResourceName(host);
  request.setNumContainers(numContainers);
  Priority prio = recordFactory.newRecordInstance(Priority.class);
  prio.setPriority(priority);
  request.setPriority(prio);
  return request;
}
Example 12
Source File: Application.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void updateResourceRequest(ResourceRequest request) {
  request.setNumContainers(request.getNumContainers() - 1);

  // Note this for next interaction with ResourceManager
  ask.remove(request);
  ask.add(
      org.apache.hadoop.yarn.server.utils.BuilderUtils.newResourceRequest(
      request)); // clone to ensure the RM doesn't manipulate the same obj

  if(LOG.isDebugEnabled()) {
    LOG.debug("updateResourceRequest:" + " application=" + applicationId
      + " request=" + request);
  }
}
Example 13
Source File: Application.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private synchronized void addResourceRequest(
    Priority priority, Map<String, ResourceRequest> requests, 
    String resourceName, Resource capability) {
  ResourceRequest request = requests.get(resourceName);
  if (request == null) {
    request = 
      org.apache.hadoop.yarn.server.utils.BuilderUtils.newResourceRequest(
          priority, resourceName, capability, 1);
    requests.put(resourceName, request);
  } else {
    request.setNumContainers(request.getNumContainers() + 1);
  }
  
  // Note this down for next interaction with ResourceManager
  ask.remove(request);
  ask.add(
      org.apache.hadoop.yarn.server.utils.BuilderUtils.newResourceRequest(
          request)); // clone to ensure the RM doesn't manipulate the same obj
  
  if(LOG.isDebugEnabled()) {
    LOG.debug("addResourceRequest: applicationId=" + applicationId.getId()
      + " priority=" + priority.getPriority()
      + " resourceName=" + resourceName + " capability=" + capability
      + " numContainers=" + request.getNumContainers()
      + " #asks=" + ask.size());
  }
}
Example 14
Source File: AppSchedulingInfo.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void decResourceRequest(String resourceName, Priority priority,
    ResourceRequest request) {
  request.setNumContainers(request.getNumContainers() - 1);
  if (request.getNumContainers() == 0) {
    requests.get(priority).remove(resourceName);
  }
}
Example 15
Source File: BuilderUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static ResourceRequest newResourceRequest(ResourceRequest r) {
  ResourceRequest request = recordFactory
      .newRecordInstance(ResourceRequest.class);
  request.setPriority(r.getPriority());
  request.setResourceName(r.getResourceName());
  request.setCapability(r.getCapability());
  request.setNumContainers(r.getNumContainers());
  return request;
}
Example 16
Source File: BuilderUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static ResourceRequest newResourceRequest(Priority priority,
    String hostName, Resource capability, int numContainers, String label) {
  ResourceRequest request =
      recordFactory.newRecordInstance(ResourceRequest.class);
  request.setPriority(priority);
  request.setResourceName(hostName);
  request.setCapability(capability);
  request.setNumContainers(numContainers);
  request.setNodeLabelExpression(label);
  return request;
}
Example 17
Source File: TestYarnApiClasses.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Simple test Resource request.
 * Test hashCode, equals and compare.
 */
@Test
public void testResourceRequest() {

  Resource resource = recordFactory.newRecordInstance(Resource.class);
  Priority priority = recordFactory.newRecordInstance(Priority.class);

  ResourceRequest original = ResourceRequest.newInstance(priority, "localhost", resource, 2) ;

  ResourceRequest copy = ResourceRequest.newInstance(priority, "localhost", resource, 2);

  assertTrue(original.equals(copy));
  assertEquals(0, original.compareTo(copy));
  assertTrue(original.hashCode() == copy.hashCode());

  copy.setNumContainers(1);

  assertFalse(original.equals(copy));
  assertNotSame(0, original.compareTo(copy));
  assertFalse(original.hashCode() == copy.hashCode());

}
Example 18
Source File: RMContainerRequestor.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private void decResourceRequest(Priority priority, String resourceName,
    Resource capability) {
  Map<String, Map<Resource, ResourceRequest>> remoteRequests =
    this.remoteRequestsTable.get(priority);
  Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
  if (reqMap == null) {
    // as we modify the resource requests by filtering out blacklisted hosts 
    // when they are added, this value may be null when being 
    // decremented
    if (LOG.isDebugEnabled()) {
      LOG.debug("Not decrementing resource as " + resourceName
          + " is not present in request table");
    }
    return;
  }
  ResourceRequest remoteRequest = reqMap.get(capability);

  if (LOG.isDebugEnabled()) {
    LOG.debug("BEFORE decResourceRequest:" + " applicationId="
        + applicationId.getId() + " priority=" + priority.getPriority()
        + " resourceName=" + resourceName + " numContainers="
        + remoteRequest.getNumContainers() + " #asks=" + ask.size());
  }

  if(remoteRequest.getNumContainers() > 0) {
    // based on blacklisting comments above we can end up decrementing more 
    // than requested. so guard for that.
    remoteRequest.setNumContainers(remoteRequest.getNumContainers() -1);
  }
  
  if (remoteRequest.getNumContainers() == 0) {
    reqMap.remove(capability);
    if (reqMap.size() == 0) {
      remoteRequests.remove(resourceName);
    }
    if (remoteRequests.size() == 0) {
      remoteRequestsTable.remove(priority);
    }
  }

  // send the updated resource request to RM
  // send 0 container count requests also to cancel previous requests
  addResourceRequestToAsk(remoteRequest);

  if (LOG.isDebugEnabled()) {
    LOG.info("AFTER decResourceRequest:" + " applicationId="
        + applicationId.getId() + " priority=" + priority.getPriority()
        + " resourceName=" + resourceName + " numContainers="
        + remoteRequest.getNumContainers() + " #asks=" + ask.size());
  }
}
Example 19
Source File: RMContainerRequestor.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private void decResourceRequest(Priority priority, String resourceName,
    Resource capability) {
  Map<String, Map<Resource, ResourceRequest>> remoteRequests =
    this.remoteRequestsTable.get(priority);
  Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
  if (reqMap == null) {
    // as we modify the resource requests by filtering out blacklisted hosts 
    // when they are added, this value may be null when being 
    // decremented
    if (LOG.isDebugEnabled()) {
      LOG.debug("Not decrementing resource as " + resourceName
          + " is not present in request table");
    }
    return;
  }
  ResourceRequest remoteRequest = reqMap.get(capability);

  if (LOG.isDebugEnabled()) {
    LOG.debug("BEFORE decResourceRequest:" + " applicationId="
        + applicationId.getId() + " priority=" + priority.getPriority()
        + " resourceName=" + resourceName + " numContainers="
        + remoteRequest.getNumContainers() + " #asks=" + ask.size());
  }

  if(remoteRequest.getNumContainers() > 0) {
    // based on blacklisting comments above we can end up decrementing more 
    // than requested. so guard for that.
    remoteRequest.setNumContainers(remoteRequest.getNumContainers() -1);
  }
  
  if (remoteRequest.getNumContainers() == 0) {
    reqMap.remove(capability);
    if (reqMap.size() == 0) {
      remoteRequests.remove(resourceName);
    }
    if (remoteRequests.size() == 0) {
      remoteRequestsTable.remove(priority);
    }
  }

  // send the updated resource request to RM
  // send 0 container count requests also to cancel previous requests
  addResourceRequestToAsk(remoteRequest);

  if (LOG.isDebugEnabled()) {
    LOG.info("AFTER decResourceRequest:" + " applicationId="
        + applicationId.getId() + " priority=" + priority.getPriority()
        + " resourceName=" + resourceName + " numContainers="
        + remoteRequest.getNumContainers() + " #asks=" + ask.size());
  }
}
Example 20
Source File: TestYarnApiClasses.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Simple test Resource request.
 * Test hashCode, equals and compare.
 */
@Test
public void testResourceRequest() {

  Resource resource = recordFactory.newRecordInstance(Resource.class);
  Priority priority = recordFactory.newRecordInstance(Priority.class);

  ResourceRequest original = ResourceRequest.newInstance(priority, "localhost", resource, 2) ;

  ResourceRequest copy = ResourceRequest.newInstance(priority, "localhost", resource, 2);

  assertTrue(original.equals(copy));
  assertEquals(0, original.compareTo(copy));
  assertTrue(original.hashCode() == copy.hashCode());

  copy.setNumContainers(1);

  assertFalse(original.equals(copy));
  assertNotSame(0, original.compareTo(copy));
  assertFalse(original.hashCode() == copy.hashCode());

}