org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest Java Examples
The following examples show how to use
org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestAllocateRequest.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testAllcoateRequestWithIncrease() { List<ContainerResourceIncreaseRequest> incRequests = new ArrayList<ContainerResourceIncreaseRequest>(); for (int i = 0; i < 3; i++) { incRequests.add(ContainerResourceIncreaseRequest.newInstance(null, Resource.newInstance(0, i))); } AllocateRequest r = AllocateRequest.newInstance(123, 0f, null, null, null, incRequests); // serde AllocateRequestProto p = ((AllocateRequestPBImpl) r).getProto(); r = new AllocateRequestPBImpl(p); // check value Assert.assertEquals(123, r.getResponseId()); Assert.assertEquals(incRequests.size(), r.getIncreaseRequests().size()); for (int i = 0; i < incRequests.size(); i++) { Assert.assertEquals(r.getIncreaseRequests().get(i).getCapability() .getVirtualCores(), incRequests.get(i).getCapability() .getVirtualCores()); } }
Example #2
Source File: TestRMContainerAllocator.java From hadoop with Apache License 2.0 | 6 votes |
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { lastAsk = request.getAskList(); for (ResourceRequest req : lastAsk) { if (ResourceRequest.ANY.equals(req.getResourceName())) { Priority priority = req.getPriority(); if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) { lastAnyAskMap = req.getNumContainers(); } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){ lastAnyAskReduce = req.getNumContainers(); } } } AllocateResponse response = AllocateResponse.newInstance( request.getResponseId(), containersToComplete, containersToAllocate, Collections.<NodeReport>emptyList(), Resource.newInstance(512000, 1024, 1024), null, 10, null, Collections.<NMToken>emptyList()); containersToComplete.clear(); containersToAllocate.clear(); return response; }
Example #3
Source File: TestRMContainerAllocator.java From big-c with Apache License 2.0 | 6 votes |
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { lastAsk = request.getAskList(); for (ResourceRequest req : lastAsk) { if (ResourceRequest.ANY.equals(req.getResourceName())) { Priority priority = req.getPriority(); if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) { lastAnyAskMap = req.getNumContainers(); } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){ lastAnyAskReduce = req.getNumContainers(); } } } AllocateResponse response = AllocateResponse.newInstance( request.getResponseId(), containersToComplete, containersToAllocate, Collections.<NodeReport>emptyList(), Resource.newInstance(512000, 1024), null, 10, null, Collections.<NMToken>emptyList()); containersToComplete.clear(); containersToAllocate.clear(); return response; }
Example #4
Source File: TestLocalContainerAllocator.java From big-c with Apache License 2.0 | 6 votes |
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { Assert.assertEquals("response ID mismatch", responseId, request.getResponseId()); ++responseId; org.apache.hadoop.yarn.api.records.Token yarnToken = null; if (amToken != null) { yarnToken = org.apache.hadoop.yarn.api.records.Token.newInstance( amToken.getIdentifier(), amToken.getKind().toString(), amToken.getPassword(), amToken.getService().toString()); } return AllocateResponse.newInstance(responseId, Collections.<ContainerStatus>emptyList(), Collections.<Container>emptyList(), Collections.<NodeReport>emptyList(), Resources.none(), null, 1, null, Collections.<NMToken>emptyList(), yarnToken, Collections.<ContainerResourceIncrease>emptyList(), Collections.<ContainerResourceDecrease>emptyList()); }
Example #5
Source File: TestLocalContainerAllocator.java From hadoop with Apache License 2.0 | 6 votes |
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { Assert.assertEquals("response ID mismatch", responseId, request.getResponseId()); ++responseId; org.apache.hadoop.yarn.api.records.Token yarnToken = null; if (amToken != null) { yarnToken = org.apache.hadoop.yarn.api.records.Token.newInstance( amToken.getIdentifier(), amToken.getKind().toString(), amToken.getPassword(), amToken.getService().toString()); } return AllocateResponse.newInstance(responseId, Collections.<ContainerStatus>emptyList(), Collections.<Container>emptyList(), Collections.<NodeReport>emptyList(), Resources.none(), null, 1, null, Collections.<NMToken>emptyList(), yarnToken, Collections.<ContainerResourceIncrease>emptyList(), Collections.<ContainerResourceDecrease>emptyList()); }
Example #6
Source File: TestAllocateRequest.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testAllcoateRequestWithIncrease() { List<ContainerResourceIncreaseRequest> incRequests = new ArrayList<ContainerResourceIncreaseRequest>(); for (int i = 0; i < 3; i++) { incRequests.add(ContainerResourceIncreaseRequest.newInstance(null, Resource.newInstance(0, i))); } AllocateRequest r = AllocateRequest.newInstance(123, 0f, null, null, null, incRequests); // serde AllocateRequestProto p = ((AllocateRequestPBImpl) r).getProto(); r = new AllocateRequestPBImpl(p); // check value Assert.assertEquals(123, r.getResponseId()); Assert.assertEquals(incRequests.size(), r.getIncreaseRequests().size()); for (int i = 0; i < incRequests.size(); i++) { Assert.assertEquals(r.getIncreaseRequests().get(i).getCapability() .getVirtualCores(), incRequests.get(i).getCapability() .getVirtualCores()); } }
Example #7
Source File: MRAMSimulator.java From hadoop with Apache License 2.0 | 5 votes |
/** * send out request for AM container */ protected void requestAMContainer() throws YarnException, IOException, InterruptedException { List<ResourceRequest> ask = new ArrayList<ResourceRequest>(); ResourceRequest amRequest = createResourceRequest( BuilderUtils.newResource(MR_AM_CONTAINER_RESOURCE_MEMORY_MB, MR_AM_CONTAINER_RESOURCE_VCORES), ResourceRequest.ANY, 1, 1); ask.add(amRequest); LOG.debug(MessageFormat.format("Application {0} sends out allocate " + "request for its AM", appId)); final AllocateRequest request = this.createAllocateRequest(ask); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(appAttemptId.toString()); Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps() .get(appAttemptId.getApplicationId()) .getRMAppAttempt(appAttemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); AllocateResponse response = ugi.doAs( new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return rm.getApplicationMasterService().allocate(request); } }); if (response != null) { responseQueue.put(response); } }
Example #8
Source File: ApplicationMasterProtocolPBClientImpl.java From big-c with Apache License 2.0 | 5 votes |
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { AllocateRequestProto requestProto = ((AllocateRequestPBImpl) request).getProto(); try { return new AllocateResponsePBImpl(proxy.allocate(null, requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); return null; } }
Example #9
Source File: ApplicationMasterProtocolPBClientImpl.java From hadoop with Apache License 2.0 | 5 votes |
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { AllocateRequestProto requestProto = ((AllocateRequestPBImpl) request).getProto(); try { return new AllocateResponsePBImpl(proxy.allocate(null, requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); return null; } }
Example #10
Source File: TestAllocateRequest.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testAllcoateRequestWithoutIncrease() { AllocateRequest r = AllocateRequest.newInstance(123, 0f, null, null, null, null); // serde AllocateRequestProto p = ((AllocateRequestPBImpl) r).getProto(); r = new AllocateRequestPBImpl(p); // check value Assert.assertEquals(123, r.getResponseId()); Assert.assertEquals(0, r.getIncreaseRequests().size()); }
Example #11
Source File: TestAMRMRPCResponseId.java From big-c with Apache License 2.0 | 5 votes |
private AllocateResponse allocate(ApplicationAttemptId attemptId, final AllocateRequest req) throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(attemptId.getApplicationId()) .getRMAppAttempt(attemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return amService.allocate(req); } }); }
Example #12
Source File: TestAMRMRPCNodeUpdates.java From big-c with Apache License 2.0 | 5 votes |
private AllocateResponse allocate(final ApplicationAttemptId attemptId, final AllocateRequest req) throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(attemptId.getApplicationId()) .getRMAppAttempt(attemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return amService.allocate(req); } }); }
Example #13
Source File: MockAM.java From big-c with Apache License 2.0 | 5 votes |
public AllocateResponse allocate( List<ResourceRequest> resourceRequest, List<ContainerId> releases) throws Exception { final AllocateRequest req = AllocateRequest.newInstance(0, 0F, resourceRequest, releases, null); return allocate(req); }
Example #14
Source File: MockAM.java From big-c with Apache License 2.0 | 5 votes |
public AllocateResponse allocate(AllocateRequest allocateRequest) throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); Token<AMRMTokenIdentifier> token = context.getRMApps().get(attemptId.getApplicationId()) .getRMAppAttempt(attemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); lastResponse = doAllocateAs(ugi, allocateRequest); return lastResponse; }
Example #15
Source File: MockAM.java From big-c with Apache License 2.0 | 5 votes |
public AllocateResponse doAllocateAs(UserGroupInformation ugi, final AllocateRequest req) throws Exception { req.setResponseId(++responseId); try { return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return amRMProtocol.allocate(req); } }); } catch (UndeclaredThrowableException e) { throw (Exception) e.getCause(); } }
Example #16
Source File: ProtocolHATestBase.java From big-c with Apache License 2.0 | 5 votes |
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { resetStartFailoverFlag(true); // make sure failover has been triggered Assert.assertTrue(waittingForFailOver()); return createFakeAllocateResponse(); }
Example #17
Source File: TestApplicationMasterServiceProtocolOnHA.java From big-c with Apache License 2.0 | 5 votes |
@Test(timeout = 15000) public void testAllocateOnHA() throws YarnException, IOException { AllocateRequest request = AllocateRequest.newInstance(0, 50f, new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>(), ResourceBlacklistRequest.newInstance(new ArrayList<String>(), new ArrayList<String>())); AllocateResponse response = amClient.allocate(request); Assert.assertEquals(response, this.cluster.createFakeAllocateResponse()); }
Example #18
Source File: AMSimulator.java From big-c with Apache License 2.0 | 5 votes |
protected AllocateRequest createAllocateRequest(List<ResourceRequest> ask, List<ContainerId> toRelease) { AllocateRequest allocateRequest = recordFactory.newRecordInstance(AllocateRequest.class); allocateRequest.setResponseId(RESPONSE_ID ++); allocateRequest.setAskList(ask); allocateRequest.setReleaseList(toRelease); return allocateRequest; }
Example #19
Source File: MRAMSimulator.java From big-c with Apache License 2.0 | 5 votes |
/** * send out request for AM container */ protected void requestAMContainer() throws YarnException, IOException, InterruptedException { List<ResourceRequest> ask = new ArrayList<ResourceRequest>(); ResourceRequest amRequest = createResourceRequest( BuilderUtils.newResource(MR_AM_CONTAINER_RESOURCE_MEMORY_MB, MR_AM_CONTAINER_RESOURCE_VCORES), ResourceRequest.ANY, 1, 1); ask.add(amRequest); LOG.debug(MessageFormat.format("Application {0} sends out allocate " + "request for its AM", appId)); final AllocateRequest request = this.createAllocateRequest(ask); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(appAttemptId.toString()); Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps() .get(appAttemptId.getApplicationId()) .getRMAppAttempt(appAttemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); AllocateResponse response = ugi.doAs( new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return rm.getApplicationMasterService().allocate(request); } }); if (response != null) { responseQueue.put(response); } }
Example #20
Source File: AMSimulator.java From hadoop with Apache License 2.0 | 5 votes |
protected AllocateRequest createAllocateRequest(List<ResourceRequest> ask, List<ContainerId> toRelease) { AllocateRequest allocateRequest = recordFactory.newRecordInstance(AllocateRequest.class); allocateRequest.setResponseId(RESPONSE_ID ++); allocateRequest.setAskList(ask); allocateRequest.setReleaseList(toRelease); return allocateRequest; }
Example #21
Source File: TestAllocateRequest.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testAllcoateRequestWithoutIncrease() { AllocateRequest r = AllocateRequest.newInstance(123, 0f, null, null, null, null); // serde AllocateRequestProto p = ((AllocateRequestPBImpl) r).getProto(); r = new AllocateRequestPBImpl(p); // check value Assert.assertEquals(123, r.getResponseId()); Assert.assertEquals(0, r.getIncreaseRequests().size()); }
Example #22
Source File: TestAMRMRPCResponseId.java From hadoop with Apache License 2.0 | 5 votes |
private AllocateResponse allocate(ApplicationAttemptId attemptId, final AllocateRequest req) throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(attemptId.getApplicationId()) .getRMAppAttempt(attemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return amService.allocate(req); } }); }
Example #23
Source File: TestAMRMRPCNodeUpdates.java From hadoop with Apache License 2.0 | 5 votes |
private AllocateResponse allocate(final ApplicationAttemptId attemptId, final AllocateRequest req) throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(attemptId.getApplicationId()) .getRMAppAttempt(attemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return amService.allocate(req); } }); }
Example #24
Source File: MockAM.java From hadoop with Apache License 2.0 | 5 votes |
public AllocateResponse allocate( List<ResourceRequest> resourceRequest, List<ContainerId> releases) throws Exception { final AllocateRequest req = AllocateRequest.newInstance(0, 0F, resourceRequest, releases, null); return allocate(req); }
Example #25
Source File: MockAM.java From hadoop with Apache License 2.0 | 5 votes |
public AllocateResponse allocate(AllocateRequest allocateRequest) throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); Token<AMRMTokenIdentifier> token = context.getRMApps().get(attemptId.getApplicationId()) .getRMAppAttempt(attemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); lastResponse = doAllocateAs(ugi, allocateRequest); return lastResponse; }
Example #26
Source File: MockAM.java From hadoop with Apache License 2.0 | 5 votes |
public AllocateResponse doAllocateAs(UserGroupInformation ugi, final AllocateRequest req) throws Exception { req.setResponseId(++responseId); try { return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return amRMProtocol.allocate(req); } }); } catch (UndeclaredThrowableException e) { throw (Exception) e.getCause(); } }
Example #27
Source File: TestApplicationMasterServiceProtocolOnHA.java From hadoop with Apache License 2.0 | 5 votes |
@Test(timeout = 15000) public void testAllocateOnHA() throws YarnException, IOException { AllocateRequest request = AllocateRequest.newInstance(0, 50f, new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>(), ResourceBlacklistRequest.newInstance(new ArrayList<String>(), new ArrayList<String>())); AllocateResponse response = amClient.allocate(request); Assert.assertEquals(response, this.cluster.createFakeAllocateResponse()); }
Example #28
Source File: ProtocolHATestBase.java From hadoop with Apache License 2.0 | 5 votes |
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { resetStartFailoverFlag(true); // make sure failover has been triggered Assert.assertTrue(waittingForFailOver()); return createFakeAllocateResponse(); }
Example #29
Source File: RMContainerRequestor.java From hadoop with Apache License 2.0 | 4 votes |
protected AllocateResponse makeRemoteRequest() throws YarnException, IOException { applyRequestLimits(); ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<String>(blacklistAdditions), new ArrayList<String>(blacklistRemovals)); AllocateRequest allocateRequest = AllocateRequest.newInstance(lastResponseID, super.getApplicationProgress(), new ArrayList<ResourceRequest>(ask), new ArrayList<ContainerId>(release), blacklistRequest); AllocateResponse allocateResponse = scheduler.allocate(allocateRequest); lastResponseID = allocateResponse.getResponseId(); availableResources = allocateResponse.getAvailableResources(); lastClusterNmCount = clusterNmCount; clusterNmCount = allocateResponse.getNumClusterNodes(); int numCompletedContainers = allocateResponse.getCompletedContainersStatuses().size(); if (ask.size() > 0 || release.size() > 0) { LOG.info("getResources() for " + applicationId + ":" + " ask=" + ask.size() + " release= " + release.size() + " newContainers=" + allocateResponse.getAllocatedContainers().size() + " finishedContainers=" + numCompletedContainers + " resourcelimit=" + availableResources + " knownNMs=" + clusterNmCount); } ask.clear(); release.clear(); if (numCompletedContainers > 0) { // re-send limited requests when a container completes to trigger asking // for more containers requestLimitsToUpdate.addAll(requestLimits.keySet()); } if (blacklistAdditions.size() > 0 || blacklistRemovals.size() > 0) { LOG.info("Update the blacklist for " + applicationId + ": blacklistAdditions=" + blacklistAdditions.size() + " blacklistRemovals=" + blacklistRemovals.size()); } blacklistAdditions.clear(); blacklistRemovals.clear(); return allocateResponse; }
Example #30
Source File: MRAMSimulator.java From big-c with Apache License 2.0 | 4 votes |
@Override protected void sendContainerRequest() throws YarnException, IOException, InterruptedException { if (isFinished) { return; } // send out request List<ResourceRequest> ask = null; if (isAMContainerRunning) { if (mapFinished != mapTotal) { // map phase if (! pendingMaps.isEmpty()) { ask = packageRequests(pendingMaps, PRIORITY_MAP); LOG.debug(MessageFormat.format("Application {0} sends out " + "request for {1} mappers.", appId, pendingMaps.size())); scheduledMaps.addAll(pendingMaps); pendingMaps.clear(); } else if (! pendingFailedMaps.isEmpty() && scheduledMaps.isEmpty()) { ask = packageRequests(pendingFailedMaps, PRIORITY_MAP); LOG.debug(MessageFormat.format("Application {0} sends out " + "requests for {1} failed mappers.", appId, pendingFailedMaps.size())); scheduledMaps.addAll(pendingFailedMaps); pendingFailedMaps.clear(); } } else if (reduceFinished != reduceTotal) { // reduce phase if (! pendingReduces.isEmpty()) { ask = packageRequests(pendingReduces, PRIORITY_REDUCE); LOG.debug(MessageFormat.format("Application {0} sends out " + "requests for {1} reducers.", appId, pendingReduces.size())); scheduledReduces.addAll(pendingReduces); pendingReduces.clear(); } else if (! pendingFailedReduces.isEmpty() && scheduledReduces.isEmpty()) { ask = packageRequests(pendingFailedReduces, PRIORITY_REDUCE); LOG.debug(MessageFormat.format("Application {0} sends out " + "request for {1} failed reducers.", appId, pendingFailedReduces.size())); scheduledReduces.addAll(pendingFailedReduces); pendingFailedReduces.clear(); } } } if (ask == null) { ask = new ArrayList<ResourceRequest>(); } final AllocateRequest request = createAllocateRequest(ask); if (totalContainers == 0) { request.setProgress(1.0f); } else { request.setProgress((float) finishedContainers / totalContainers); } UserGroupInformation ugi = UserGroupInformation.createRemoteUser(appAttemptId.toString()); Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps() .get(appAttemptId.getApplicationId()) .getRMAppAttempt(appAttemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); AllocateResponse response = ugi.doAs( new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return rm.getApplicationMasterService().allocate(request); } }); if (response != null) { responseQueue.put(response); } }