Java Code Examples for org.apache.hadoop.yarn.api.records.Container#getResource()
The following examples show how to use
org.apache.hadoop.yarn.api.records.Container#getResource() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: NMSimulator.java From hadoop with Apache License 2.0 | 6 votes |
|
/**
* launch a new container with the given life time
*/
public void addNewContainer(Container container, long lifeTimeMS) {
LOG.debug(MessageFormat.format("NodeManager {0} launches a new " +
"container ({1}).", node.getNodeID(), container.getId()));
if (lifeTimeMS != -1) {
// normal container
ContainerSimulator cs = new ContainerSimulator(container.getId(),
container.getResource(), lifeTimeMS + System.currentTimeMillis(),
lifeTimeMS);
containerQueue.add(cs);
runningContainers.put(cs.getId(), cs);
} else {
// AM container
// -1 means AMContainer
synchronized(amContainerList) {
amContainerList.add(container.getId());
}
}
}
Example 2
| Source File: NMSimulator.java From big-c with Apache License 2.0 | 6 votes |
|
/**
* launch a new container with the given life time
*/
public void addNewContainer(Container container, long lifeTimeMS) {
LOG.debug(MessageFormat.format("NodeManager {0} launches a new " +
"container ({1}).", node.getNodeID(), container.getId()));
if (lifeTimeMS != -1) {
// normal container
ContainerSimulator cs = new ContainerSimulator(container.getId(),
container.getResource(), lifeTimeMS + System.currentTimeMillis(),
lifeTimeMS);
containerQueue.add(cs);
runningContainers.put(cs.getId(), cs);
} else {
// AM container
// -1 means AMContainer
synchronized(amContainerList) {
amContainerList.add(container.getId());
}
}
}
Example 3
| Source File: YarnTaskSchedulerService.java From incubator-tez with Apache License 2.0 | 6 votes |
|
private CookieContainerRequest getMatchingRequestWithPriority(
Container container,
String location) {
Priority priority = container.getPriority();
Resource capability = container.getResource();
List<? extends Collection<CookieContainerRequest>> requestsList =
amRmClient.getMatchingRequests(priority, location, capability);
if (!requestsList.isEmpty()) {
// pick first one
for (Collection<CookieContainerRequest> requests : requestsList) {
for (CookieContainerRequest cookieContainerRequest : requests) {
if (canAssignTaskToContainer(cookieContainerRequest, container)) {
return cookieContainerRequest;
}
}
}
}
return null;
}
Example 4
| Source File: YarnTaskSchedulerService.java From tez with Apache License 2.0 | 6 votes |
|
private CookieContainerRequest getMatchingRequestWithPriority(
Container container,
String location) {
Priority priority = container.getPriority();
Resource capability = container.getResource();
List<? extends Collection<CookieContainerRequest>> requestsList =
amRmClient.getMatchingRequests(priority, location, capability);
if (!requestsList.isEmpty()) {
// pick first one
for (Collection<CookieContainerRequest> requests : requestsList) {
for (CookieContainerRequest cookieContainerRequest : requests) {
if (canAssignTaskToContainer(cookieContainerRequest, container)) {
return cookieContainerRequest;
}
}
}
}
return null;
}
Example 5
| Source File: FSAppAttempt.java From hadoop with Apache License 2.0 | 4 votes |
|
/**
* Assign a container to this node to facilitate {@code request}. If node does
* not have enough memory, create a reservation. This is called once we are
* sure the particular request should be facilitated by this node.
*
* @param node
* The node to try placing the container on.
* @param request
* The ResourceRequest we're trying to satisfy.
* @param type
* The locality of the assignment.
* @param reserved
* Whether there's already a container reserved for this app on the node.
* @return
* If an assignment was made, returns the resources allocated to the
* container. If a reservation was made, returns
* FairScheduler.CONTAINER_RESERVED. If no assignment or reservation was
* made, returns an empty resource.
*/
private Resource assignContainer(
FSSchedulerNode node, ResourceRequest request, NodeType type,
boolean reserved) {
// How much does this request need?
Resource capability = request.getCapability();
// How much does the node have?
Resource available = node.getAvailableResource();
Container container = null;
if (reserved) {
container = node.getReservedContainer().getContainer();
} else {
container = createContainer(node, capability, request.getPriority());
}
// Can we allocate a container on this node?
if (Resources.fitsIn(capability, available)) {
// Inform the application of the new container for this request
RMContainer allocatedContainer =
allocate(type, node, request.getPriority(), request, container);
if (allocatedContainer == null) {
// Did the application need this resource?
if (reserved) {
unreserve(request.getPriority(), node);
}
return Resources.none();
}
// If we had previously made a reservation, delete it
if (reserved) {
unreserve(request.getPriority(), node);
}
// Inform the node
node.allocateContainer(allocatedContainer);
// If this container is used to run AM, update the leaf queue's AM usage
if (getLiveContainers().size() == 1 && !getUnmanagedAM()) {
getQueue().addAMResourceUsage(container.getResource());
setAmRunning(true);
}
return container.getResource();
} else {
if (!FairScheduler.fitsInMaxShare(getQueue(), capability)) {
return Resources.none();
}
// The desired container won't fit here, so reserve
reserve(request.getPriority(), node, container, reserved);
return FairScheduler.CONTAINER_RESERVED;
}
}
Example 6
| Source File: FSAppAttempt.java From big-c with Apache License 2.0 | 4 votes |
|
/**
* Assign a container to this node to facilitate {@code request}. If node does
* not have enough memory, create a reservation. This is called once we are
* sure the particular request should be facilitated by this node.
*
* @param node
* The node to try placing the container on.
* @param request
* The ResourceRequest we're trying to satisfy.
* @param type
* The locality of the assignment.
* @param reserved
* Whether there's already a container reserved for this app on the node.
* @return
* If an assignment was made, returns the resources allocated to the
* container. If a reservation was made, returns
* FairScheduler.CONTAINER_RESERVED. If no assignment or reservation was
* made, returns an empty resource.
*/
private Resource assignContainer(
FSSchedulerNode node, ResourceRequest request, NodeType type,
boolean reserved) {
// How much does this request need?
Resource capability = request.getCapability();
// How much does the node have?
Resource available = node.getAvailableResource();
Container container = null;
if (reserved) {
container = node.getReservedContainer().getContainer();
} else {
container = createContainer(node, capability, request.getPriority());
}
// Can we allocate a container on this node?
if (Resources.fitsIn(capability, available)) {
// Inform the application of the new container for this request
RMContainer allocatedContainer =
allocate(type, node, request.getPriority(), request, container);
if (allocatedContainer == null) {
// Did the application need this resource?
if (reserved) {
unreserve(request.getPriority(), node);
}
return Resources.none();
}
// If we had previously made a reservation, delete it
if (reserved) {
unreserve(request.getPriority(), node);
}
// Inform the node
node.allocateContainer(allocatedContainer);
// If this container is used to run AM, update the leaf queue's AM usage
if (getLiveContainers().size() == 1 && !getUnmanagedAM()) {
getQueue().addAMResourceUsage(container.getResource());
setAmRunning(true);
}
return container.getResource();
} else {
if (!FairScheduler.fitsInMaxShare(getQueue(), capability)) {
return Resources.none();
}
// The desired container won't fit here, so reserve
reserve(request.getPriority(), node, container, reserved);
return FairScheduler.CONTAINER_RESERVED;
}
}
Example 7
| Source File: LeafQueue.java From big-c with Apache License 2.0 | 4 votes |
|
@Override
public void completedContainer(Resource clusterResource,
FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer,
ContainerStatus containerStatus, RMContainerEventType event, CSQueue childQueue,
boolean sortQueues) {
if (application != null) {
boolean removed = false;
// Careful! Locking order is important!
synchronized (this) {
Container container = rmContainer.getContainer();
Resource toRelease = null;
// Inform the application & the node
// Note: It's safe to assume that all state changes to RMContainer
// happen under scheduler's lock...
// So, this is, in effect, a transaction across application & node
if (rmContainer.getState() == RMContainerState.RESERVED) {
removed = unreserve(application, rmContainer.getReservedPriority(),
node, rmContainer);
toRelease = container.getResource();
} else {
//for container suspend event
if (event == RMContainerEventType.SUSPEND){
removed =
application.containerSuspend(rmContainer, containerStatus, event);
//suspend and resume in fifo order
if(!suspendedApps.contains(application.getApplicationAttemptId())){
LOG.info(application.getApplicationAttemptId()+"into suspending list");
suspendedApps.add(application.getApplicationAttemptId());
}
//we suspend the container on this node
node.suspendContainer(container,rmContainer.getLastPreemptedResource());
toRelease = rmContainer.getLastPreemptedResource();
}else{
toRelease = rmContainer.getCurrentUsedResource();
removed =
application.containerCompleted(rmContainer, containerStatus, event);
node.releaseContainer(container,toRelease);
//for container suspend event
//in case of completing a suspended container
}
}
// Book-keeping
if (removed) {
//更新资源试用情况
if(event == RMContainerEventType.SUSPEND){
releaseResource(clusterResource, application,
toRelease, node.getLabels(),true);
}else{
releaseResource(clusterResource, application,
toRelease, node.getLabels(),false);
}
LOG.info("completedContainer" +
" container=" + container +
" queue=" + this +
" cluster=" + clusterResource);
}
}
if (removed) {
// Inform the parent queue _outside_ of the leaf-queue lock
getParent().completedContainer(clusterResource, application, node,
rmContainer, null, event, this, sortQueues);
}
}
}
Example 8
| Source File: YarnTaskSchedulerService.java From incubator-tez with Apache License 2.0 | 4 votes |
|
private CookieContainerRequest getMatchingRequestWithoutPriority(
Container container,
String location,
boolean considerContainerAffinity) {
Resource capability = container.getResource();
List<? extends Collection<CookieContainerRequest>> pRequestsList =
amRmClient.getMatchingRequestsForTopPriority(location, capability);
if (considerContainerAffinity &&
!priorityHasAffinity.contains(amRmClient.getTopPriority())) {
considerContainerAffinity = false;
}
if (pRequestsList == null || pRequestsList.isEmpty()) {
return null;
}
CookieContainerRequest firstMatch = null;
for (Collection<CookieContainerRequest> requests : pRequestsList) {
for (CookieContainerRequest cookieContainerRequest : requests) {
if (firstMatch == null || // we dont have a match. So look for one
// we have a match but are looking for a better container level match.
// skip the expensive canAssignTaskToContainer() if the request is
// not affinitized to the container
container.getId().equals(cookieContainerRequest.getAffinitizedContainer())
) {
if (canAssignTaskToContainer(cookieContainerRequest, container)) {
// request matched to container
if (!considerContainerAffinity) {
return cookieContainerRequest;
}
ContainerId affCId = cookieContainerRequest.getAffinitizedContainer();
boolean canMatchTaskWithAffinity = true;
if (affCId == null ||
!heldContainers.containsKey(affCId) ||
inUseContainers.contains(affCId)) {
// affinity not specified
// affinitized container is no longer held
// affinitized container is in use
canMatchTaskWithAffinity = false;
}
if (canMatchTaskWithAffinity) {
if (container.getId().equals(
cookieContainerRequest.getAffinitizedContainer())) {
// container level match
if (LOG.isDebugEnabled()) {
LOG.debug("Matching with affinity for request: "
+ cookieContainerRequest + " container: " + affCId);
}
return cookieContainerRequest;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Skipping request for container " + container.getId()
+ " due to affinity. Request: " + cookieContainerRequest
+ " affContainer: " + affCId);
}
} else {
firstMatch = cookieContainerRequest;
}
}
}
}
}
return firstMatch;
}
Example 9
| Source File: YarnTaskSchedulerService.java From tez with Apache License 2.0 | 4 votes |
|
private CookieContainerRequest getMatchingRequestWithoutPriority(
Container container,
String location,
boolean considerContainerAffinity) {
Resource capability = container.getResource();
List<? extends Collection<CookieContainerRequest>> pRequestsList =
amRmClient.getMatchingRequestsForTopPriority(location, capability);
if (considerContainerAffinity &&
!priorityHasAffinity.contains(amRmClient.getTopPriority())) {
considerContainerAffinity = false;
}
if (pRequestsList == null || pRequestsList.isEmpty()) {
return null;
}
CookieContainerRequest firstMatch = null;
for (Collection<CookieContainerRequest> requests : pRequestsList) {
for (CookieContainerRequest cookieContainerRequest : requests) {
if (firstMatch == null || // we dont have a match. So look for one
// we have a match but are looking for a better container level match.
// skip the expensive canAssignTaskToContainer() if the request is
// not affinitized to the container
container.getId().equals(cookieContainerRequest.getAffinitizedContainer())
) {
if (canAssignTaskToContainer(cookieContainerRequest, container)) {
// request matched to container
if (!considerContainerAffinity) {
return cookieContainerRequest;
}
ContainerId affCId = cookieContainerRequest.getAffinitizedContainer();
boolean canMatchTaskWithAffinity = true;
if (affCId == null ||
!heldContainers.containsKey(affCId) ||
inUseContainers.contains(affCId)) {
// affinity not specified
// affinitized container is no longer held
// affinitized container is in use
canMatchTaskWithAffinity = false;
}
if (canMatchTaskWithAffinity) {
if (container.getId().equals(
cookieContainerRequest.getAffinitizedContainer())) {
// container level match
if (LOG.isDebugEnabled()) {
LOG.debug("Matching with affinity for request: "
+ cookieContainerRequest + " container: " + affCId);
}
return cookieContainerRequest;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Skipping request for container " + container.getId()
+ " due to affinity. Request: " + cookieContainerRequest
+ " affContainer: " + affCId);
}
} else {
firstMatch = cookieContainerRequest;
}
}
}
}
}
return firstMatch;
}
