org.apache.hadoop.yarn.factories.RecordFactory Java Examples
The following examples show how to use
org.apache.hadoop.yarn.factories.RecordFactory.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestAppManager.java From big-c with Apache License 2.0 | 6 votes |
@SuppressWarnings("deprecation") @Before public void setUp() { long now = System.currentTimeMillis(); rmContext = mockRMContext(1, now - 10); ResourceScheduler scheduler = mockResourceScheduler(); Configuration conf = new Configuration(); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); appMonitor = new TestRMAppManager(rmContext, new ClientToAMTokenSecretManagerInRM(), scheduler, masterService, new ApplicationACLsManager(conf), conf); appId = MockApps.newAppID(1); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); asContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); asContext.setApplicationId(appId); asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory)); asContext.setResource(mockResource()); setupDispatcher(rmContext, conf); }
Example #2
Source File: TestClientRMService.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testGetApplicationAttempts() throws YarnException, IOException { ClientRMService rmService = createRMService(); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetApplicationAttemptsRequest request = recordFactory .newRecordInstance(GetApplicationAttemptsRequest.class); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(123456, 1), 1); request.setApplicationId(ApplicationId.newInstance(123456, 1)); try { GetApplicationAttemptsResponse response = rmService .getApplicationAttempts(request); Assert.assertEquals(1, response.getApplicationAttemptList().size()); Assert.assertEquals(attemptId, response.getApplicationAttemptList() .get(0).getApplicationAttemptId()); } catch (ApplicationNotFoundException ex) { Assert.fail(ex.getMessage()); } }
Example #3
Source File: TestClientRMService.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testGetApplicationAttemptReport() throws YarnException, IOException { ClientRMService rmService = createRMService(); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetApplicationAttemptReportRequest request = recordFactory .newRecordInstance(GetApplicationAttemptReportRequest.class); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(123456, 1), 1); request.setApplicationAttemptId(attemptId); try { GetApplicationAttemptReportResponse response = rmService .getApplicationAttemptReport(request); Assert.assertEquals(attemptId, response.getApplicationAttemptReport() .getApplicationAttemptId()); } catch (ApplicationNotFoundException ex) { Assert.fail(ex.getMessage()); } }
Example #4
Source File: TestClientRMService.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testGetContainerReport() throws YarnException, IOException { ClientRMService rmService = createRMService(); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetContainerReportRequest request = recordFactory .newRecordInstance(GetContainerReportRequest.class); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(123456, 1), 1); ContainerId containerId = ContainerId.newContainerId(attemptId, 1); request.setContainerId(containerId); try { GetContainerReportResponse response = rmService .getContainerReport(request); Assert.assertEquals(containerId, response.getContainerReport() .getContainerId()); } catch (ApplicationNotFoundException ex) { Assert.fail(ex.getMessage()); } }
Example #5
Source File: TestClientRMService.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testNonExistingApplicationReport() throws YarnException { RMContext rmContext = mock(RMContext.class); when(rmContext.getRMApps()).thenReturn( new ConcurrentHashMap<ApplicationId, RMApp>()); ClientRMService rmService = new ClientRMService(rmContext, null, null, null, null, null); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetApplicationReportRequest request = recordFactory .newRecordInstance(GetApplicationReportRequest.class); request.setApplicationId(ApplicationId.newInstance(0, 0)); try { rmService.getApplicationReport(request); Assert.fail(); } catch (ApplicationNotFoundException ex) { Assert.assertEquals(ex.getMessage(), "Application with id '" + request.getApplicationId() + "' doesn't exist in RM."); } }
Example #6
Source File: TestClientRMService.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testGetContainers() throws YarnException, IOException { ClientRMService rmService = createRMService(); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetContainersRequest request = recordFactory .newRecordInstance(GetContainersRequest.class); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(123456, 1), 1); ContainerId containerId = ContainerId.newContainerId(attemptId, 1); request.setApplicationAttemptId(attemptId); try { GetContainersResponse response = rmService.getContainers(request); Assert.assertEquals(containerId, response.getContainerList().get(0) .getContainerId()); } catch (ApplicationNotFoundException ex) { Assert.fail(ex.getMessage()); } }
Example #7
Source File: TestAppManager.java From hadoop with Apache License 2.0 | 6 votes |
@SuppressWarnings("deprecation") @Before public void setUp() { long now = System.currentTimeMillis(); rmContext = mockRMContext(1, now - 10); ResourceScheduler scheduler = mockResourceScheduler(); Configuration conf = new Configuration(); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); appMonitor = new TestRMAppManager(rmContext, new ClientToAMTokenSecretManagerInRM(), scheduler, masterService, new ApplicationACLsManager(conf), conf); appId = MockApps.newAppID(1); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); asContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); asContext.setApplicationId(appId); asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory)); asContext.setResource(mockResource()); setupDispatcher(rmContext, conf); }
Example #8
Source File: TestClientRMService.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testNonExistingApplicationReport() throws YarnException { RMContext rmContext = mock(RMContext.class); when(rmContext.getRMApps()).thenReturn( new ConcurrentHashMap<ApplicationId, RMApp>()); ClientRMService rmService = new ClientRMService(rmContext, null, null, null, null, null); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetApplicationReportRequest request = recordFactory .newRecordInstance(GetApplicationReportRequest.class); request.setApplicationId(ApplicationId.newInstance(0, 0)); try { rmService.getApplicationReport(request); Assert.fail(); } catch (ApplicationNotFoundException ex) { Assert.assertEquals(ex.getMessage(), "Application with id '" + request.getApplicationId() + "' doesn't exist in RM."); } }
Example #9
Source File: TestClientRMService.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testGetApplicationAttemptReport() throws YarnException, IOException { ClientRMService rmService = createRMService(); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetApplicationAttemptReportRequest request = recordFactory .newRecordInstance(GetApplicationAttemptReportRequest.class); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(123456, 1), 1); request.setApplicationAttemptId(attemptId); try { GetApplicationAttemptReportResponse response = rmService .getApplicationAttemptReport(request); Assert.assertEquals(attemptId, response.getApplicationAttemptReport() .getApplicationAttemptId()); } catch (ApplicationNotFoundException ex) { Assert.fail(ex.getMessage()); } }
Example #10
Source File: TestClientRMService.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testGetApplicationAttempts() throws YarnException, IOException { ClientRMService rmService = createRMService(); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetApplicationAttemptsRequest request = recordFactory .newRecordInstance(GetApplicationAttemptsRequest.class); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(123456, 1), 1); request.setApplicationId(ApplicationId.newInstance(123456, 1)); try { GetApplicationAttemptsResponse response = rmService .getApplicationAttempts(request); Assert.assertEquals(1, response.getApplicationAttemptList().size()); Assert.assertEquals(attemptId, response.getApplicationAttemptList() .get(0).getApplicationAttemptId()); } catch (ApplicationNotFoundException ex) { Assert.fail(ex.getMessage()); } }
Example #11
Source File: TestClientRMService.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testGetContainerReport() throws YarnException, IOException { ClientRMService rmService = createRMService(); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetContainerReportRequest request = recordFactory .newRecordInstance(GetContainerReportRequest.class); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(123456, 1), 1); ContainerId containerId = ContainerId.newContainerId(attemptId, 1); request.setContainerId(containerId); try { GetContainerReportResponse response = rmService .getContainerReport(request); Assert.assertEquals(containerId, response.getContainerReport() .getContainerId()); } catch (ApplicationNotFoundException ex) { Assert.fail(ex.getMessage()); } }
Example #12
Source File: ContainerLocalizer.java From hadoop with Apache License 2.0 | 6 votes |
public ContainerLocalizer(FileContext lfs, String user, String appId, String localizerId, List<Path> localDirs, RecordFactory recordFactory) throws IOException { if (null == user) { throw new IOException("Cannot initialize for null user"); } if (null == localizerId) { throw new IOException("Cannot initialize for null containerId"); } this.lfs = lfs; this.user = user; this.appId = appId; this.localDirs = localDirs; this.localizerId = localizerId; this.recordFactory = recordFactory; this.conf = new Configuration(); this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId); this.pendingResources = new HashMap<LocalResource,Future<Path>>(); }
Example #13
Source File: TestClientRMService.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testGetContainers() throws YarnException, IOException { ClientRMService rmService = createRMService(); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetContainersRequest request = recordFactory .newRecordInstance(GetContainersRequest.class); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(123456, 1), 1); ContainerId containerId = ContainerId.newContainerId(attemptId, 1); request.setApplicationAttemptId(attemptId); try { GetContainersResponse response = rmService.getContainers(request); Assert.assertEquals(containerId, response.getContainerList().get(0) .getContainerId()); } catch (ApplicationNotFoundException ex) { Assert.fail(ex.getMessage()); } }
Example #14
Source File: ContainerLocalizer.java From big-c with Apache License 2.0 | 6 votes |
public ContainerLocalizer(FileContext lfs, String user, String appId, String localizerId, List<Path> localDirs, RecordFactory recordFactory) throws IOException { if (null == user) { throw new IOException("Cannot initialize for null user"); } if (null == localizerId) { throw new IOException("Cannot initialize for null containerId"); } this.lfs = lfs; this.user = user; this.appId = appId; this.localDirs = localDirs; this.localizerId = localizerId; this.recordFactory = recordFactory; this.conf = new Configuration(); this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId); this.pendingResources = new HashMap<LocalResource,Future<Path>>(); }
Example #15
Source File: MiniYARNClusterSplice.java From spliceengine with GNU Affero General Public License v3.0 | 6 votes |
@Override protected NodeStatusUpdater createNodeStatusUpdater(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker) { return new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics) { @Override protected ResourceTracker getRMClient() { final ResourceTrackerService rt = getResourceManager().getResourceTrackerService(); final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); // For in-process communication without RPC return Utils.getResourceTracker(rt); } @Override protected void stopRMProxy() { } }; }
Example #16
Source File: TestTezLocalCacheManager.java From tez with Apache License 2.0 | 6 votes |
private static LocalResource createFile(String content) throws IOException { FileContext fs = FileContext.getLocalFSFileContext(); java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt"); File temp = tempFile.toFile(); temp.deleteOnExit(); Path p = new Path("file:///" + tempFile.toAbsolutePath().toString()); Files.write(tempFile, content.getBytes()); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p); ret.setResource(yarnUrlFromPath); ret.setSize(content.getBytes().length); ret.setType(LocalResourceType.FILE); ret.setVisibility(LocalResourceVisibility.PRIVATE); ret.setTimestamp(fs.getFileStatus(p).getModificationTime()); return ret; }
Example #17
Source File: TestRecordFactory.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testPbRecordFactory() { RecordFactory pbRecordFactory = RecordFactoryPBImpl.get(); try { LocalizerHeartbeatResponse response = pbRecordFactory.newRecordInstance( LocalizerHeartbeatResponse.class); Assert.assertEquals(LocalizerHeartbeatResponsePBImpl.class, response.getClass()); } catch (YarnRuntimeException e) { e.printStackTrace(); Assert.fail("Failed to crete record"); } }
Example #18
Source File: NodeManager.java From hadoop with Apache License 2.0 | 5 votes |
public static org.apache.hadoop.yarn.server.api.records.NodeStatus createNodeStatus(NodeId nodeId, List<ContainerStatus> containers) { RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class); nodeStatus.setNodeId(nodeId); nodeStatus.setContainersStatuses(containers); NodeHealthStatus nodeHealthStatus = recordFactory.newRecordInstance(NodeHealthStatus.class); nodeHealthStatus.setIsNodeHealthy(true); nodeStatus.setNodeHealthStatus(nodeHealthStatus); return nodeStatus; }
Example #19
Source File: TestAppManager.java From big-c with Apache License 2.0 | 5 votes |
private static ContainerLaunchContext mockContainerLaunchContext( RecordFactory recordFactory) { ContainerLaunchContext amContainer = recordFactory.newRecordInstance( ContainerLaunchContext.class); amContainer.setApplicationACLs(new HashMap<ApplicationAccessType, String>());; return amContainer; }
Example #20
Source File: TestUtils.java From big-c with Apache License 2.0 | 5 votes |
public static ResourceRequest createResourceRequest( String resourceName, int memory, int numContainers, boolean relaxLocality, Priority priority, RecordFactory recordFactory) { ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); Resource capability = Resources.createResource(memory, 1); request.setNumContainers(numContainers); request.setResourceName(resourceName); request.setCapability(capability); request.setRelaxLocality(relaxLocality); request.setPriority(priority); return request; }
Example #21
Source File: TestYSCRecordFactory.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testPbRecordFactory() { RecordFactory pbRecordFactory = RecordFactoryPBImpl.get(); try { NodeHeartbeatRequest request = pbRecordFactory.newRecordInstance(NodeHeartbeatRequest.class); Assert.assertEquals(NodeHeartbeatRequestPBImpl.class, request.getClass()); } catch (YarnRuntimeException e) { e.printStackTrace(); Assert.fail("Failed to crete record"); } }
Example #22
Source File: TestTaskAttemptListenerImpl.java From hadoop with Apache License 2.0 | 5 votes |
private static TaskAttemptCompletionEvent createTce(int eventId, boolean isMap, TaskAttemptCompletionEventStatus status) { JobId jid = MRBuilderUtils.newJobId(12345, 1, 1); TaskId tid = MRBuilderUtils.newTaskId(jid, 0, isMap ? org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP : org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE); TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); TaskAttemptCompletionEvent tce = recordFactory .newRecordInstance(TaskAttemptCompletionEvent.class); tce.setEventId(eventId); tce.setAttemptId(attemptId); tce.setStatus(status); return tce; }
Example #23
Source File: RecordFactoryProvider.java From hadoop with Apache License 2.0 | 5 votes |
public static RecordFactory getRecordFactory(Configuration conf) { if (conf == null) { //Assuming the default configuration has the correct factories set. //Users can specify a particular factory by providing a configuration. conf = defaultConf; } String recordFactoryClassName = conf.get( YarnConfiguration.IPC_RECORD_FACTORY_CLASS, YarnConfiguration.DEFAULT_IPC_RECORD_FACTORY_CLASS); return (RecordFactory) getFactoryClassInstance(recordFactoryClassName); }
Example #24
Source File: MockApp.java From big-c with Apache License 2.0 | 5 votes |
public MockApp(String user, long clusterTimeStamp, int uniqId) { super(); this.user = user; // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider .getRecordFactory(new Configuration()); this.appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, uniqId); appState = ApplicationState.NEW; }
Example #25
Source File: TestClientRMService.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testGetApplicationReport() throws Exception { YarnScheduler yarnScheduler = mock(YarnScheduler.class); RMContext rmContext = mock(RMContext.class); mockRMContext(yarnScheduler, rmContext); ApplicationId appId1 = getApplicationId(1); ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class); when( mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true); ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, null, mockAclsManager, null, null); try { RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetApplicationReportRequest request = recordFactory .newRecordInstance(GetApplicationReportRequest.class); request.setApplicationId(appId1); GetApplicationReportResponse response = rmService.getApplicationReport(request); ApplicationReport report = response.getApplicationReport(); ApplicationResourceUsageReport usageReport = report.getApplicationResourceUsageReport(); Assert.assertEquals(10, usageReport.getMemorySeconds()); Assert.assertEquals(3, usageReport.getVcoreSeconds()); Assert.assertEquals(3, usageReport.getGcoreSeconds()); } finally { rmService.close(); } }
Example #26
Source File: TestUtils.java From hadoop with Apache License 2.0 | 5 votes |
public static ResourceRequest createResourceRequest( String resourceName, int memory, int numContainers, boolean relaxLocality, Priority priority, RecordFactory recordFactory) { ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); Resource capability = Resources.createResource(memory, 1); request.setNumContainers(numContainers); request.setResourceName(resourceName); request.setCapability(capability); request.setRelaxLocality(relaxLocality); request.setPriority(priority); return request; }
Example #27
Source File: TestAppManager.java From hadoop with Apache License 2.0 | 5 votes |
private static ContainerLaunchContext mockContainerLaunchContext( RecordFactory recordFactory) { ContainerLaunchContext amContainer = recordFactory.newRecordInstance( ContainerLaunchContext.class); amContainer.setApplicationACLs(new HashMap<ApplicationAccessType, String>());; return amContainer; }
Example #28
Source File: NodeManager.java From big-c with Apache License 2.0 | 5 votes |
public static org.apache.hadoop.yarn.server.api.records.NodeStatus createNodeStatus(NodeId nodeId, List<ContainerStatus> containers) { RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class); nodeStatus.setNodeId(nodeId); nodeStatus.setContainersStatuses(containers); NodeHealthStatus nodeHealthStatus = recordFactory.newRecordInstance(NodeHealthStatus.class); nodeHealthStatus.setIsNodeHealthy(true); nodeStatus.setNodeHealthStatus(nodeHealthStatus); return nodeStatus; }
Example #29
Source File: TestRecordFactory.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testPbRecordFactory() { RecordFactory pbRecordFactory = RecordFactoryPBImpl.get(); try { LocalizerHeartbeatResponse response = pbRecordFactory.newRecordInstance( LocalizerHeartbeatResponse.class); Assert.assertEquals(LocalizerHeartbeatResponsePBImpl.class, response.getClass()); } catch (YarnRuntimeException e) { e.printStackTrace(); Assert.fail("Failed to crete record"); } }
Example #30
Source File: TestClientRMService.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testGetApplicationReport() throws Exception { YarnScheduler yarnScheduler = mock(YarnScheduler.class); RMContext rmContext = mock(RMContext.class); mockRMContext(yarnScheduler, rmContext); ApplicationId appId1 = getApplicationId(1); ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class); when( mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true); ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, null, mockAclsManager, null, null); try { RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); GetApplicationReportRequest request = recordFactory .newRecordInstance(GetApplicationReportRequest.class); request.setApplicationId(appId1); GetApplicationReportResponse response = rmService.getApplicationReport(request); ApplicationReport report = response.getApplicationReport(); ApplicationResourceUsageReport usageReport = report.getApplicationResourceUsageReport(); Assert.assertEquals(10, usageReport.getMemorySeconds()); Assert.assertEquals(3, usageReport.getVcoreSeconds()); } finally { rmService.close(); } }