org.apache.hadoop.yarn.server.resourcemanager.ClientRMService Java Examples
The following examples show how to use
org.apache.hadoop.yarn.server.resourcemanager.ClientRMService.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: TestDelegationTokenRenewer.java From hadoop with Apache License 2.0 | 6 votes |
|
@Before
public void setUp() throws Exception {
counter = new AtomicInteger(0);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
UserGroupInformation.setConfiguration(conf);
eventQueue = new LinkedBlockingQueue<Event>();
dispatcher = new AsyncDispatcher(eventQueue);
Renewer.reset();
delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
RMContext mockContext = mock(RMContext.class);
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getSystemCredentialsForApps()).thenReturn(
new ConcurrentHashMap<ApplicationId, ByteBuffer>());
when(mockContext.getDelegationTokenRenewer()).thenReturn(
delegationTokenRenewer);
when(mockContext.getDispatcher()).thenReturn(dispatcher);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
InetSocketAddress sockAddr =
InetSocketAddress.createUnresolved("localhost", 1234);
when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
delegationTokenRenewer.setRMContext(mockContext);
delegationTokenRenewer.init(conf);
delegationTokenRenewer.start();
}
Example #2
| Source File: TestDelegationTokenRenewer.java From big-c with Apache License 2.0 | 6 votes |
|
@Before
public void setUp() throws Exception {
counter = new AtomicInteger(0);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
UserGroupInformation.setConfiguration(conf);
eventQueue = new LinkedBlockingQueue<Event>();
dispatcher = new AsyncDispatcher(eventQueue);
Renewer.reset();
delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
RMContext mockContext = mock(RMContext.class);
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getSystemCredentialsForApps()).thenReturn(
new ConcurrentHashMap<ApplicationId, ByteBuffer>());
when(mockContext.getDelegationTokenRenewer()).thenReturn(
delegationTokenRenewer);
when(mockContext.getDispatcher()).thenReturn(dispatcher);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
InetSocketAddress sockAddr =
InetSocketAddress.createUnresolved("localhost", 1234);
when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
delegationTokenRenewer.setRMContext(mockContext);
delegationTokenRenewer.init(conf);
delegationTokenRenewer.start();
}
Example #3
| Source File: ProtocolHATestBase.java From hadoop with Apache License 2.0 | 5 votes |
|
@Override
protected ResourceManager createResourceManager() {
return new ResourceManager() {
@Override
protected void doSecureLogin() throws IOException {
// Don't try to login using keytab in the testcases.
}
@Override
protected ClientRMService createClientRMService() {
if (overrideClientRMService) {
return new CustomedClientRMService(this.rmContext, this.scheduler,
this.rmAppManager, this.applicationACLsManager,
this.queueACLsManager,
this.rmContext.getRMDelegationTokenSecretManager());
}
return super.createClientRMService();
}
@Override
protected ResourceTrackerService createResourceTrackerService() {
if (overrideRTS) {
return new CustomedResourceTrackerService(this.rmContext,
this.nodesListManager, this.nmLivelinessMonitor,
this.rmContext.getContainerTokenSecretManager(),
this.rmContext.getNMTokenSecretManager());
}
return super.createResourceTrackerService();
}
@Override
protected ApplicationMasterService createApplicationMasterService() {
if (overrideApplicationMasterService) {
return new CustomedApplicationMasterService(this.rmContext,
this.scheduler);
}
return super.createApplicationMasterService();
}
};
}
Example #4
| Source File: TestRMWebAppFairScheduler.java From big-c with Apache License 2.0 | 5 votes |
|
private static ResourceManager mockRm(RMContext rmContext) throws
IOException {
ResourceManager rm = mock(ResourceManager.class);
ResourceScheduler rs = mockFairScheduler();
ClientRMService clientRMService = mockClientRMService(rmContext);
when(rm.getResourceScheduler()).thenReturn(rs);
when(rm.getRMContext()).thenReturn(rmContext);
when(rm.getClientRMService()).thenReturn(clientRMService);
return rm;
}
Example #5
| Source File: TestRMWebApp.java From big-c with Apache License 2.0 | 5 votes |
|
public static ClientRMService mockClientRMService(RMContext rmContext) {
ClientRMService clientRMService = mock(ClientRMService.class);
List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
for (RMApp app : rmContext.getRMApps().values()) {
ApplicationReport appReport =
ApplicationReport.newInstance(
app.getApplicationId(), (ApplicationAttemptId) null,
app.getUser(), app.getQueue(),
app.getName(), (String) null, 0, (Token) null,
app.createApplicationState(),
app.getDiagnostics().toString(), (String) null,
app.getStartTime(), app.getFinishTime(),
app.getFinalApplicationStatus(),
(ApplicationResourceUsageReport) null, app.getTrackingUrl(),
app.getProgress(), app.getApplicationType(), (Token) null);
appReports.add(appReport);
}
GetApplicationsResponse response = mock(GetApplicationsResponse.class);
when(response.getApplicationList()).thenReturn(appReports);
try {
when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
.thenReturn(response);
} catch (YarnException e) {
Assert.fail("Exception is not expteced.");
}
return clientRMService;
}
Example #6
| Source File: TestRMWebApp.java From big-c with Apache License 2.0 | 5 votes |
|
public static ResourceManager mockRm(RMContext rmContext) throws IOException {
ResourceManager rm = mock(ResourceManager.class);
ResourceScheduler rs = mockCapacityScheduler();
ApplicationACLsManager aclMgr = mockAppACLsManager();
ClientRMService clientRMService = mockClientRMService(rmContext);
when(rm.getResourceScheduler()).thenReturn(rs);
when(rm.getRMContext()).thenReturn(rmContext);
when(rm.getApplicationACLsManager()).thenReturn(aclMgr);
when(rm.getClientRMService()).thenReturn(clientRMService);
return rm;
}
Example #7
| Source File: TestRMWebServices.java From big-c with Apache License 2.0 | 5 votes |
|
@Test
public void testAppsRace() throws Exception {
// mock up an RM that returns app reports for apps that don't exist
// in the RMApps list
ApplicationId appId = ApplicationId.newInstance(1, 1);
ApplicationReport mockReport = mock(ApplicationReport.class);
when(mockReport.getApplicationId()).thenReturn(appId);
GetApplicationsResponse mockAppsResponse =
mock(GetApplicationsResponse.class);
when(mockAppsResponse.getApplicationList())
.thenReturn(Arrays.asList(new ApplicationReport[] { mockReport }));
ClientRMService mockClientSvc = mock(ClientRMService.class);
when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class),
anyBoolean())).thenReturn(mockAppsResponse);
ResourceManager mockRM = mock(ResourceManager.class);
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
null, null, null, null, null);
when(mockRM.getRMContext()).thenReturn(rmContext);
when(mockRM.getClientRMService()).thenReturn(mockClientSvc);
RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(),
mock(HttpServletResponse.class));
final Set<String> emptySet =
Collections.unmodifiableSet(Collections.<String>emptySet());
// verify we don't get any apps when querying
HttpServletRequest mockHsr = mock(HttpServletRequest.class);
AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
null, null, null, null, null, null, null, emptySet, emptySet);
assertTrue(appsInfo.getApps().isEmpty());
// verify we don't get an NPE when specifying a final status query
appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
null, null, null, null, null, null, null, emptySet, emptySet);
assertTrue(appsInfo.getApps().isEmpty());
}
Example #8
| Source File: TestDelegationTokenRenewerLifecycle.java From big-c with Apache License 2.0 | 5 votes |
|
@Test
public void testStartupFailure() throws Exception {
Configuration conf = new Configuration();
DelegationTokenRenewer delegationTokenRenewer =
new DelegationTokenRenewer();
RMContext mockContext = mock(RMContext.class);
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
delegationTokenRenewer.setRMContext(mockContext);
delegationTokenRenewer.init(conf);
delegationTokenRenewer.stop();
}
Example #9
| Source File: TestRMWebAppFairScheduler.java From big-c with Apache License 2.0 | 5 votes |
|
private static ResourceManager mockRmWithApps(RMContext rmContext) throws
IOException {
ResourceManager rm = mock(ResourceManager.class);
ResourceScheduler rs = mockFairSchedulerWithoutApps(rmContext);
ClientRMService clientRMService = mockClientRMService(rmContext);
when(rm.getResourceScheduler()).thenReturn(rs);
when(rm.getRMContext()).thenReturn(rmContext);
when(rm.getClientRMService()).thenReturn(clientRMService);
return rm;
}
Example #10
| Source File: TestDelegationTokenRenewer.java From big-c with Apache License 2.0 | 5 votes |
|
@Test(timeout=20000)
public void testDTRonAppSubmission()
throws IOException, InterruptedException, BrokenBarrierException {
final Credentials credsx = new Credentials();
final Token<DelegationTokenIdentifier> tokenx = mock(Token.class);
when(tokenx.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
DelegationTokenIdentifier dtId1 =
new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"),
new Text("user1"));
when(tokenx.decodeIdentifier()).thenReturn(dtId1);
credsx.addToken(new Text("token"), tokenx);
doReturn(true).when(tokenx).isManaged();
doThrow(new IOException("boom"))
.when(tokenx).renew(any(Configuration.class));
// fire up the renewer
final DelegationTokenRenewer dtr =
createNewDelegationTokenRenewer(conf, counter);
RMContext mockContext = mock(RMContext.class);
when(mockContext.getSystemCredentialsForApps()).thenReturn(
new ConcurrentHashMap<ApplicationId, ByteBuffer>());
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
InetSocketAddress sockAddr =
InetSocketAddress.createUnresolved("localhost", 1234);
when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
dtr.setRMContext(mockContext);
when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
dtr.init(conf);
dtr.start();
try {
dtr.addApplicationSync(mock(ApplicationId.class), credsx, false, "user");
fail("Catch IOException on app submission");
} catch (IOException e){
Assert.assertTrue(e.getMessage().contains(tokenx.toString()));
Assert.assertTrue(e.getCause().toString().contains("boom"));
}
}
Example #11
| Source File: TwillTester.java From twill with Apache License 2.0 | 5 votes |
|
public ApplicationResourceUsageReport getApplicationResourceReport(String appId) throws Exception {
List<String> splits = Lists.newArrayList(Splitter.on('_').split(appId));
Preconditions.checkArgument(splits.size() == 3, "Invalid application id - " + appId);
ApplicationId applicationId = ApplicationId.newInstance(Long.parseLong(splits.get(1)),
Integer.parseInt(splits.get(2)));
ClientRMService clientRMService = cluster.getResourceManager().getClientRMService();
GetApplicationReportRequest request = Records.newRecord(GetApplicationReportRequest.class);
request.setApplicationId(applicationId);
return clientRMService.getApplicationReport(request)
.getApplicationReport().getApplicationResourceUsageReport();
}
Example #12
| Source File: ProtocolHATestBase.java From big-c with Apache License 2.0 | 5 votes |
|
@Override
protected ResourceManager createResourceManager() {
return new ResourceManager() {
@Override
protected void doSecureLogin() throws IOException {
// Don't try to login using keytab in the testcases.
}
@Override
protected ClientRMService createClientRMService() {
if (overrideClientRMService) {
return new CustomedClientRMService(this.rmContext, this.scheduler,
this.rmAppManager, this.applicationACLsManager,
this.queueACLsManager,
this.rmContext.getRMDelegationTokenSecretManager());
}
return super.createClientRMService();
}
@Override
protected ResourceTrackerService createResourceTrackerService() {
if (overrideRTS) {
return new CustomedResourceTrackerService(this.rmContext,
this.nodesListManager, this.nmLivelinessMonitor,
this.rmContext.getContainerTokenSecretManager(),
this.rmContext.getNMTokenSecretManager());
}
return super.createResourceTrackerService();
}
@Override
protected ApplicationMasterService createApplicationMasterService() {
if (overrideApplicationMasterService) {
return new CustomedApplicationMasterService(this.rmContext,
this.scheduler);
}
return super.createApplicationMasterService();
}
};
}
Example #13
| Source File: TestRMWebAppFairScheduler.java From hadoop with Apache License 2.0 | 5 votes |
|
private static ResourceManager mockRmWithApps(RMContext rmContext) throws
IOException {
ResourceManager rm = mock(ResourceManager.class);
ResourceScheduler rs = mockFairSchedulerWithoutApps(rmContext);
ClientRMService clientRMService = mockClientRMService(rmContext);
when(rm.getResourceScheduler()).thenReturn(rs);
when(rm.getRMContext()).thenReturn(rmContext);
when(rm.getClientRMService()).thenReturn(clientRMService);
return rm;
}
Example #14
| Source File: TestRMWebAppFairScheduler.java From hadoop with Apache License 2.0 | 5 votes |
|
private static ResourceManager mockRm(RMContext rmContext) throws
IOException {
ResourceManager rm = mock(ResourceManager.class);
ResourceScheduler rs = mockFairScheduler();
ClientRMService clientRMService = mockClientRMService(rmContext);
when(rm.getResourceScheduler()).thenReturn(rs);
when(rm.getRMContext()).thenReturn(rmContext);
when(rm.getClientRMService()).thenReturn(clientRMService);
return rm;
}
Example #15
| Source File: TestRMWebApp.java From hadoop with Apache License 2.0 | 5 votes |
|
public static ClientRMService mockClientRMService(RMContext rmContext) {
ClientRMService clientRMService = mock(ClientRMService.class);
List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
for (RMApp app : rmContext.getRMApps().values()) {
ApplicationReport appReport =
ApplicationReport.newInstance(
app.getApplicationId(), (ApplicationAttemptId) null,
app.getUser(), app.getQueue(),
app.getName(), (String) null, 0, (Token) null,
app.createApplicationState(),
app.getDiagnostics().toString(), (String) null,
app.getStartTime(), app.getFinishTime(),
app.getFinalApplicationStatus(),
(ApplicationResourceUsageReport) null, app.getTrackingUrl(),
app.getProgress(), app.getApplicationType(), (Token) null);
appReports.add(appReport);
}
GetApplicationsResponse response = mock(GetApplicationsResponse.class);
when(response.getApplicationList()).thenReturn(appReports);
try {
when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
.thenReturn(response);
} catch (YarnException e) {
Assert.fail("Exception is not expteced.");
}
return clientRMService;
}
Example #16
| Source File: TestRMWebApp.java From hadoop with Apache License 2.0 | 5 votes |
|
public static ResourceManager mockRm(RMContext rmContext) throws IOException {
ResourceManager rm = mock(ResourceManager.class);
ResourceScheduler rs = mockCapacityScheduler();
ApplicationACLsManager aclMgr = mockAppACLsManager();
ClientRMService clientRMService = mockClientRMService(rmContext);
when(rm.getResourceScheduler()).thenReturn(rs);
when(rm.getRMContext()).thenReturn(rmContext);
when(rm.getApplicationACLsManager()).thenReturn(aclMgr);
when(rm.getClientRMService()).thenReturn(clientRMService);
return rm;
}
Example #17
| Source File: TestRMWebServices.java From hadoop with Apache License 2.0 | 5 votes |
|
@Test
public void testAppsRace() throws Exception {
// mock up an RM that returns app reports for apps that don't exist
// in the RMApps list
ApplicationId appId = ApplicationId.newInstance(1, 1);
ApplicationReport mockReport = mock(ApplicationReport.class);
when(mockReport.getApplicationId()).thenReturn(appId);
GetApplicationsResponse mockAppsResponse =
mock(GetApplicationsResponse.class);
when(mockAppsResponse.getApplicationList())
.thenReturn(Arrays.asList(new ApplicationReport[] { mockReport }));
ClientRMService mockClientSvc = mock(ClientRMService.class);
when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class),
anyBoolean())).thenReturn(mockAppsResponse);
ResourceManager mockRM = mock(ResourceManager.class);
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
null, null, null, null, null);
when(mockRM.getRMContext()).thenReturn(rmContext);
when(mockRM.getClientRMService()).thenReturn(mockClientSvc);
RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(),
mock(HttpServletResponse.class));
final Set<String> emptySet =
Collections.unmodifiableSet(Collections.<String>emptySet());
// verify we don't get any apps when querying
HttpServletRequest mockHsr = mock(HttpServletRequest.class);
AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
null, null, null, null, null, null, null, emptySet, emptySet);
assertTrue(appsInfo.getApps().isEmpty());
// verify we don't get an NPE when specifying a final status query
appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
null, null, null, null, null, null, null, emptySet, emptySet);
assertTrue(appsInfo.getApps().isEmpty());
}
Example #18
| Source File: TestDelegationTokenRenewerLifecycle.java From hadoop with Apache License 2.0 | 5 votes |
|
@Test
public void testStartupFailure() throws Exception {
Configuration conf = new Configuration();
DelegationTokenRenewer delegationTokenRenewer =
new DelegationTokenRenewer();
RMContext mockContext = mock(RMContext.class);
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
delegationTokenRenewer.setRMContext(mockContext);
delegationTokenRenewer.init(conf);
delegationTokenRenewer.stop();
}
Example #19
| Source File: StramMiniClusterTest.java From attic-apex-core with Apache License 2.0 | 5 votes |
|
private void checkNodeState() throws YarnException
{
GetClusterNodesRequest request = Records.newRecord(GetClusterNodesRequest.class);
ClientRMService clientRMService = yarnCluster.getResourceManager().getClientRMService();
GetClusterNodesResponse response = clientRMService.getClusterNodes(request);
List<NodeReport> nodeReports = response.getNodeReports();
LOG.info("{}", nodeReports);
for (NodeReport nr: nodeReports) {
if (!nr.getNodeState().isUnusable()) {
return;
}
}
fail("Yarn Mini cluster should have at least one usable node.");
}
Example #20
| Source File: TestDelegationTokenRenewer.java From hadoop with Apache License 2.0 | 5 votes |
|
@Test(timeout=20000)
public void testDTRonAppSubmission()
throws IOException, InterruptedException, BrokenBarrierException {
final Credentials credsx = new Credentials();
final Token<DelegationTokenIdentifier> tokenx = mock(Token.class);
when(tokenx.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
DelegationTokenIdentifier dtId1 =
new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"),
new Text("user1"));
when(tokenx.decodeIdentifier()).thenReturn(dtId1);
credsx.addToken(new Text("token"), tokenx);
doReturn(true).when(tokenx).isManaged();
doThrow(new IOException("boom"))
.when(tokenx).renew(any(Configuration.class));
// fire up the renewer
final DelegationTokenRenewer dtr =
createNewDelegationTokenRenewer(conf, counter);
RMContext mockContext = mock(RMContext.class);
when(mockContext.getSystemCredentialsForApps()).thenReturn(
new ConcurrentHashMap<ApplicationId, ByteBuffer>());
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
InetSocketAddress sockAddr =
InetSocketAddress.createUnresolved("localhost", 1234);
when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
dtr.setRMContext(mockContext);
when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
dtr.init(conf);
dtr.start();
try {
dtr.addApplicationSync(mock(ApplicationId.class), credsx, false, "user");
fail("Catch IOException on app submission");
} catch (IOException e){
Assert.assertTrue(e.getMessage().contains(tokenx.toString()));
Assert.assertTrue(e.getCause().toString().contains("boom"));
}
}
Example #21
| Source File: TestDelegationTokenRenewer.java From big-c with Apache License 2.0 | 4 votes |
|
/**
* Basic idea of the test:
* 0. Setup token KEEP_ALIVE
* 1. create tokens.
* 2. register them for renewal - to be cancelled on app complete
* 3. Complete app.
* 4. Verify token is alive within the KEEP_ALIVE time
* 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
* @throws IOException
* @throws URISyntaxException
*/
@Test(timeout=60000)
public void testDTKeepAlive1 () throws Exception {
Configuration lconf = new Configuration(conf);
lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
//Keep tokens alive for 6 seconds.
lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
//Try removing tokens every second.
lconf.setLong(
YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS,
1000l);
DelegationTokenRenewer localDtr =
createNewDelegationTokenRenewer(lconf, counter);
RMContext mockContext = mock(RMContext.class);
when(mockContext.getSystemCredentialsForApps()).thenReturn(
new ConcurrentHashMap<ApplicationId, ByteBuffer>());
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
when(mockContext.getDelegationTokenRenewer()).thenReturn(
localDtr);
when(mockContext.getDispatcher()).thenReturn(dispatcher);
InetSocketAddress sockAddr =
InetSocketAddress.createUnresolved("localhost", 1234);
when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
localDtr.setRMContext(mockContext);
localDtr.init(lconf);
localDtr.start();
MyFS dfs = (MyFS)FileSystem.get(lconf);
LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode());
Credentials ts = new Credentials();
// get the delegation tokens
MyToken token1 = dfs.getDelegationToken("user1");
String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
ts.addToken(new Text(nn1), token1);
// register the tokens for renewal
ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
localDtr.addApplicationAsync(applicationId_0, ts, true, "user");
waitForEventsToGetProcessed(localDtr);
if (!eventQueue.isEmpty()){
Event evt = eventQueue.take();
if (evt instanceof RMAppEvent) {
Assert.assertEquals(((RMAppEvent)evt).getType(), RMAppEventType.START);
} else {
fail("RMAppEvent.START was expected!!");
}
}
localDtr.applicationFinished(applicationId_0);
waitForEventsToGetProcessed(localDtr);
//Token should still be around. Renewal should not fail.
token1.renew(lconf);
//Allow the keepalive time to run out
Thread.sleep(10000l);
//The token should have been cancelled at this point. Renewal will fail.
try {
token1.renew(lconf);
fail("Renewal of cancelled token should have failed");
} catch (InvalidToken ite) {}
}
Example #22
| Source File: TestRMWebAppFairScheduler.java From big-c with Apache License 2.0 | 4 votes |
|
public static ClientRMService mockClientRMService(RMContext rmContext) {
return mock(ClientRMService.class);
}
Example #23
| Source File: MockRMContext.java From incubator-myriad with Apache License 2.0 | 4 votes |
|
@Override
public ClientRMService getClientRMService() {
return clientRMService;
}
Example #24
| Source File: MockRMContext.java From incubator-myriad with Apache License 2.0 | 4 votes |
|
@Override
public void setClientRMService(ClientRMService clientRMService) {
this.clientRMService = clientRMService;
}
Example #25
| Source File: TestDelegationTokenRenewer.java From big-c with Apache License 2.0 | 4 votes |
|
@Test(timeout=20000)
public void testConcurrentAddApplication()
throws IOException, InterruptedException, BrokenBarrierException {
final CyclicBarrier startBarrier = new CyclicBarrier(2);
final CyclicBarrier endBarrier = new CyclicBarrier(2);
// this token uses barriers to block during renew
final Credentials creds1 = new Credentials();
final Token<DelegationTokenIdentifier> token1 = mock(Token.class);
when(token1.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
DelegationTokenIdentifier dtId1 =
new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"),
new Text("user1"));
when(token1.decodeIdentifier()).thenReturn(dtId1);
creds1.addToken(new Text("token"), token1);
doReturn(true).when(token1).isManaged();
doAnswer(new Answer<Long>() {
public Long answer(InvocationOnMock invocation)
throws InterruptedException, BrokenBarrierException {
startBarrier.await();
endBarrier.await();
return Long.MAX_VALUE;
}}).when(token1).renew(any(Configuration.class));
// this dummy token fakes renewing
final Credentials creds2 = new Credentials();
final Token<DelegationTokenIdentifier> token2 = mock(Token.class);
when(token2.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
when(token2.decodeIdentifier()).thenReturn(dtId1);
creds2.addToken(new Text("token"), token2);
doReturn(true).when(token2).isManaged();
doReturn(Long.MAX_VALUE).when(token2).renew(any(Configuration.class));
// fire up the renewer
final DelegationTokenRenewer dtr =
createNewDelegationTokenRenewer(conf, counter);
RMContext mockContext = mock(RMContext.class);
when(mockContext.getSystemCredentialsForApps()).thenReturn(
new ConcurrentHashMap<ApplicationId, ByteBuffer>());
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
InetSocketAddress sockAddr =
InetSocketAddress.createUnresolved("localhost", 1234);
when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
dtr.setRMContext(mockContext);
when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
dtr.init(conf);
dtr.start();
// submit a job that blocks during renewal
Thread submitThread = new Thread() {
@Override
public void run() {
dtr.addApplicationAsync(mock(ApplicationId.class), creds1, false, "user");
}
};
submitThread.start();
// wait till 1st submit blocks, then submit another
startBarrier.await();
dtr.addApplicationAsync(mock(ApplicationId.class), creds2, false, "user");
// signal 1st to complete
endBarrier.await();
submitThread.join();
}
Example #26
| Source File: TestDelegationTokenRenewer.java From big-c with Apache License 2.0 | 4 votes |
|
/**
* Basic idea of the test:
* 0. Setup token KEEP_ALIVE
* 1. create tokens.
* 2. register them for renewal - to be cancelled on app complete
* 3. Complete app.
* 4. Verify token is alive within the KEEP_ALIVE time
* 5. Send an explicity KEEP_ALIVE_REQUEST
* 6. Verify token KEEP_ALIVE time is renewed.
* 7. Verify token has been cancelled after the renewed KEEP_ALIVE_TIME.
* @throws IOException
* @throws URISyntaxException
*/
@Test(timeout=60000)
public void testDTKeepAlive2() throws Exception {
Configuration lconf = new Configuration(conf);
lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
//Keep tokens alive for 6 seconds.
lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
//Try removing tokens every second.
lconf.setLong(
YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS,
1000l);
DelegationTokenRenewer localDtr =
createNewDelegationTokenRenewer(conf, counter);
RMContext mockContext = mock(RMContext.class);
when(mockContext.getSystemCredentialsForApps()).thenReturn(
new ConcurrentHashMap<ApplicationId, ByteBuffer>());
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
when(mockContext.getDelegationTokenRenewer()).thenReturn(
localDtr);
when(mockContext.getDispatcher()).thenReturn(dispatcher);
InetSocketAddress sockAddr =
InetSocketAddress.createUnresolved("localhost", 1234);
when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
localDtr.setRMContext(mockContext);
localDtr.init(lconf);
localDtr.start();
MyFS dfs = (MyFS)FileSystem.get(lconf);
LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode());
Credentials ts = new Credentials();
// get the delegation tokens
MyToken token1 = dfs.getDelegationToken("user1");
String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
ts.addToken(new Text(nn1), token1);
// register the tokens for renewal
ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
localDtr.addApplicationAsync(applicationId_0, ts, true, "user");
localDtr.applicationFinished(applicationId_0);
waitForEventsToGetProcessed(delegationTokenRenewer);
//Send another keep alive.
localDtr.updateKeepAliveApplications(Collections
.singletonList(applicationId_0));
//Renewal should not fail.
token1.renew(lconf);
//Token should be around after this.
Thread.sleep(4500l);
//Renewal should not fail. - ~1.5 seconds for keepalive timeout.
token1.renew(lconf);
//Allow the keepalive time to run out
Thread.sleep(3000l);
//The token should have been cancelled at this point. Renewal will fail.
try {
token1.renew(lconf);
fail("Renewal of cancelled token should have failed");
} catch (InvalidToken ite) {}
}
Example #27
| Source File: TestRMWebAppFairScheduler.java From hadoop with Apache License 2.0 | 4 votes |
|
public static ClientRMService mockClientRMService(RMContext rmContext) {
return mock(ClientRMService.class);
}
Example #28
| Source File: TestDelegationTokenRenewer.java From hadoop with Apache License 2.0 | 4 votes |
|
@Test(timeout=20000)
public void testConcurrentAddApplication()
throws IOException, InterruptedException, BrokenBarrierException {
final CyclicBarrier startBarrier = new CyclicBarrier(2);
final CyclicBarrier endBarrier = new CyclicBarrier(2);
// this token uses barriers to block during renew
final Credentials creds1 = new Credentials();
final Token<DelegationTokenIdentifier> token1 = mock(Token.class);
when(token1.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
DelegationTokenIdentifier dtId1 =
new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"),
new Text("user1"));
when(token1.decodeIdentifier()).thenReturn(dtId1);
creds1.addToken(new Text("token"), token1);
doReturn(true).when(token1).isManaged();
doAnswer(new Answer<Long>() {
public Long answer(InvocationOnMock invocation)
throws InterruptedException, BrokenBarrierException {
startBarrier.await();
endBarrier.await();
return Long.MAX_VALUE;
}}).when(token1).renew(any(Configuration.class));
// this dummy token fakes renewing
final Credentials creds2 = new Credentials();
final Token<DelegationTokenIdentifier> token2 = mock(Token.class);
when(token2.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
when(token2.decodeIdentifier()).thenReturn(dtId1);
creds2.addToken(new Text("token"), token2);
doReturn(true).when(token2).isManaged();
doReturn(Long.MAX_VALUE).when(token2).renew(any(Configuration.class));
// fire up the renewer
final DelegationTokenRenewer dtr =
createNewDelegationTokenRenewer(conf, counter);
RMContext mockContext = mock(RMContext.class);
when(mockContext.getSystemCredentialsForApps()).thenReturn(
new ConcurrentHashMap<ApplicationId, ByteBuffer>());
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
InetSocketAddress sockAddr =
InetSocketAddress.createUnresolved("localhost", 1234);
when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
dtr.setRMContext(mockContext);
when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
dtr.init(conf);
dtr.start();
// submit a job that blocks during renewal
Thread submitThread = new Thread() {
@Override
public void run() {
dtr.addApplicationAsync(mock(ApplicationId.class), creds1, false, "user");
}
};
submitThread.start();
// wait till 1st submit blocks, then submit another
startBarrier.await();
dtr.addApplicationAsync(mock(ApplicationId.class), creds2, false, "user");
// signal 1st to complete
endBarrier.await();
submitThread.join();
}
Example #29
| Source File: TestDelegationTokenRenewer.java From hadoop with Apache License 2.0 | 4 votes |
|
/**
* Basic idea of the test:
* 0. Setup token KEEP_ALIVE
* 1. create tokens.
* 2. register them for renewal - to be cancelled on app complete
* 3. Complete app.
* 4. Verify token is alive within the KEEP_ALIVE time
* 5. Send an explicity KEEP_ALIVE_REQUEST
* 6. Verify token KEEP_ALIVE time is renewed.
* 7. Verify token has been cancelled after the renewed KEEP_ALIVE_TIME.
* @throws IOException
* @throws URISyntaxException
*/
@Test(timeout=60000)
public void testDTKeepAlive2() throws Exception {
Configuration lconf = new Configuration(conf);
lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
//Keep tokens alive for 6 seconds.
lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
//Try removing tokens every second.
lconf.setLong(
YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS,
1000l);
DelegationTokenRenewer localDtr =
createNewDelegationTokenRenewer(conf, counter);
RMContext mockContext = mock(RMContext.class);
when(mockContext.getSystemCredentialsForApps()).thenReturn(
new ConcurrentHashMap<ApplicationId, ByteBuffer>());
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
when(mockContext.getDelegationTokenRenewer()).thenReturn(
localDtr);
when(mockContext.getDispatcher()).thenReturn(dispatcher);
InetSocketAddress sockAddr =
InetSocketAddress.createUnresolved("localhost", 1234);
when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
localDtr.setRMContext(mockContext);
localDtr.init(lconf);
localDtr.start();
MyFS dfs = (MyFS)FileSystem.get(lconf);
LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode());
Credentials ts = new Credentials();
// get the delegation tokens
MyToken token1 = dfs.getDelegationToken("user1");
String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
ts.addToken(new Text(nn1), token1);
// register the tokens for renewal
ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
localDtr.addApplicationAsync(applicationId_0, ts, true, "user");
localDtr.applicationFinished(applicationId_0);
waitForEventsToGetProcessed(delegationTokenRenewer);
//Send another keep alive.
localDtr.updateKeepAliveApplications(Collections
.singletonList(applicationId_0));
//Renewal should not fail.
token1.renew(lconf);
//Token should be around after this.
Thread.sleep(4500l);
//Renewal should not fail. - ~1.5 seconds for keepalive timeout.
token1.renew(lconf);
//Allow the keepalive time to run out
Thread.sleep(3000l);
//The token should have been cancelled at this point. Renewal will fail.
try {
token1.renew(lconf);
fail("Renewal of cancelled token should have failed");
} catch (InvalidToken ite) {}
}
Example #30
| Source File: TestDelegationTokenRenewer.java From hadoop with Apache License 2.0 | 4 votes |
|
/**
* Basic idea of the test:
* 0. Setup token KEEP_ALIVE
* 1. create tokens.
* 2. register them for renewal - to be cancelled on app complete
* 3. Complete app.
* 4. Verify token is alive within the KEEP_ALIVE time
* 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
* @throws IOException
* @throws URISyntaxException
*/
@Test(timeout=60000)
public void testDTKeepAlive1 () throws Exception {
Configuration lconf = new Configuration(conf);
lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
//Keep tokens alive for 6 seconds.
lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
//Try removing tokens every second.
lconf.setLong(
YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS,
1000l);
DelegationTokenRenewer localDtr =
createNewDelegationTokenRenewer(lconf, counter);
RMContext mockContext = mock(RMContext.class);
when(mockContext.getSystemCredentialsForApps()).thenReturn(
new ConcurrentHashMap<ApplicationId, ByteBuffer>());
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
when(mockContext.getDelegationTokenRenewer()).thenReturn(
localDtr);
when(mockContext.getDispatcher()).thenReturn(dispatcher);
InetSocketAddress sockAddr =
InetSocketAddress.createUnresolved("localhost", 1234);
when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
localDtr.setRMContext(mockContext);
localDtr.init(lconf);
localDtr.start();
MyFS dfs = (MyFS)FileSystem.get(lconf);
LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode());
Credentials ts = new Credentials();
// get the delegation tokens
MyToken token1 = dfs.getDelegationToken("user1");
String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
ts.addToken(new Text(nn1), token1);
// register the tokens for renewal
ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
localDtr.addApplicationAsync(applicationId_0, ts, true, "user");
waitForEventsToGetProcessed(localDtr);
if (!eventQueue.isEmpty()){
Event evt = eventQueue.take();
if (evt instanceof RMAppEvent) {
Assert.assertEquals(((RMAppEvent)evt).getType(), RMAppEventType.START);
} else {
fail("RMAppEvent.START was expected!!");
}
}
localDtr.applicationFinished(applicationId_0);
waitForEventsToGetProcessed(localDtr);
//Token should still be around. Renewal should not fail.
token1.renew(lconf);
//Allow the keepalive time to run out
Thread.sleep(10000l);
//The token should have been cancelled at this point. Renewal will fail.
try {
token1.renew(lconf);
fail("Renewal of cancelled token should have failed");
} catch (InvalidToken ite) {}
}
