Java Code Examples for org.apache.hadoop.yarn.factories.RecordFactory#newRecordInstance()

The following examples show how to use org.apache.hadoop.yarn.factories.RecordFactory#newRecordInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetApplicationAttemptReport() throws YarnException,
    IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptReportRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationAttemptId(attemptId);

  try {
    GetApplicationAttemptReportResponse response = rmService
        .getApplicationAttemptReport(request);
    Assert.assertEquals(attemptId, response.getApplicationAttemptReport()
        .getApplicationAttemptId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
Example 2
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetContainerReport() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainerReportRequest request = recordFactory
      .newRecordInstance(GetContainerReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setContainerId(containerId);

  try {
    GetContainerReportResponse response = rmService
        .getContainerReport(request);
    Assert.assertEquals(containerId, response.getContainerReport()
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
Example 3
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNonExistingApplicationReport() throws YarnException {
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getRMApps()).thenReturn(
      new ConcurrentHashMap<ApplicationId, RMApp>());
  ClientRMService rmService = new ClientRMService(rmContext, null, null,
      null, null, null);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationReportRequest request = recordFactory
      .newRecordInstance(GetApplicationReportRequest.class);
  request.setApplicationId(ApplicationId.newInstance(0, 0));
  try {
    rmService.getApplicationReport(request);
    Assert.fail();
  } catch (ApplicationNotFoundException ex) {
    Assert.assertEquals(ex.getMessage(),
        "Application with id '" + request.getApplicationId()
            + "' doesn't exist in RM.");
  }
}
Example 4
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetContainers() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainersRequest request = recordFactory
      .newRecordInstance(GetContainersRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setApplicationAttemptId(attemptId);
  try {
    GetContainersResponse response = rmService.getContainers(request);
    Assert.assertEquals(containerId, response.getContainerList().get(0)
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
Example 5
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetApplicationAttempts() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptsRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptsRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationId(ApplicationId.newInstance(123456, 1));

  try {
    GetApplicationAttemptsResponse response = rmService
        .getApplicationAttempts(request);
    Assert.assertEquals(1, response.getApplicationAttemptList().size());
    Assert.assertEquals(attemptId, response.getApplicationAttemptList()
        .get(0).getApplicationAttemptId());

  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
Example 6
Source File: TestTezLocalCacheManager.java    From tez with Apache License 2.0 6 votes vote down vote up 
private static LocalResource createFile(String content) throws IOException {
  FileContext fs = FileContext.getLocalFSFileContext();

  java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt");
  File temp = tempFile.toFile();
  temp.deleteOnExit();
  Path p = new Path("file:///" + tempFile.toAbsolutePath().toString());

  Files.write(tempFile, content.getBytes());

  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p);
  ret.setResource(yarnUrlFromPath);
  ret.setSize(content.getBytes().length);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(LocalResourceVisibility.PRIVATE);
  ret.setTimestamp(fs.getFileStatus(p).getModificationTime());
  return ret;
}
Example 7
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetApplicationAttemptReport() throws YarnException,
    IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptReportRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationAttemptId(attemptId);

  try {
    GetApplicationAttemptReportResponse response = rmService
        .getApplicationAttemptReport(request);
    Assert.assertEquals(attemptId, response.getApplicationAttemptReport()
        .getApplicationAttemptId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
Example 8
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNonExistingApplicationReport() throws YarnException {
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getRMApps()).thenReturn(
      new ConcurrentHashMap<ApplicationId, RMApp>());
  ClientRMService rmService = new ClientRMService(rmContext, null, null,
      null, null, null);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationReportRequest request = recordFactory
      .newRecordInstance(GetApplicationReportRequest.class);
  request.setApplicationId(ApplicationId.newInstance(0, 0));
  try {
    rmService.getApplicationReport(request);
    Assert.fail();
  } catch (ApplicationNotFoundException ex) {
    Assert.assertEquals(ex.getMessage(),
        "Application with id '" + request.getApplicationId()
            + "' doesn't exist in RM.");
  }
}
Example 9
Source File: TestAppManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
Example 10
Source File: NodeManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static org.apache.hadoop.yarn.server.api.records.NodeStatus 
createNodeStatus(NodeId nodeId, List<ContainerStatus> containers) {
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = 
      recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class);
  nodeStatus.setNodeId(nodeId);
  nodeStatus.setContainersStatuses(containers);
  NodeHealthStatus nodeHealthStatus = 
    recordFactory.newRecordInstance(NodeHealthStatus.class);
  nodeHealthStatus.setIsNodeHealthy(true);
  nodeStatus.setNodeHealthStatus(nodeHealthStatus);
  return nodeStatus;
}
Example 11
Source File: TestUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}
Example 12
Source File: TestUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}
Example 13
Source File: NodeManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static org.apache.hadoop.yarn.server.api.records.NodeStatus 
createNodeStatus(NodeId nodeId, List<ContainerStatus> containers) {
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = 
      recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class);
  nodeStatus.setNodeId(nodeId);
  nodeStatus.setContainersStatuses(containers);
  NodeHealthStatus nodeHealthStatus = 
    recordFactory.newRecordInstance(NodeHealthStatus.class);
  nodeHealthStatus.setIsNodeHealthy(true);
  nodeStatus.setNodeHealthStatus(nodeHealthStatus);
  return nodeStatus;
}
Example 14
Source File: TestTaskAttemptListenerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private static TaskAttemptCompletionEvent createTce(int eventId,
    boolean isMap, TaskAttemptCompletionEventStatus status) {
  JobId jid = MRBuilderUtils.newJobId(12345, 1, 1);
  TaskId tid = MRBuilderUtils.newTaskId(jid, 0,
      isMap ? org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP
          : org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  TaskAttemptCompletionEvent tce = recordFactory
      .newRecordInstance(TaskAttemptCompletionEvent.class);
  tce.setEventId(eventId);
  tce.setAttemptId(attemptId);
  tce.setStatus(status);
  return tce;
}
Example 15
Source File: TestYSCRecordFactory.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testPbRecordFactory() {
  RecordFactory pbRecordFactory = RecordFactoryPBImpl.get();
  try {
    NodeHeartbeatRequest request = pbRecordFactory.newRecordInstance(NodeHeartbeatRequest.class);
    Assert.assertEquals(NodeHeartbeatRequestPBImpl.class, request.getClass());
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to crete record");
  }
  
}
Example 16
Source File: TestRecordFactory.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testPbRecordFactory() {
  RecordFactory pbRecordFactory = RecordFactoryPBImpl.get();
  
  try {
    LocalizerHeartbeatResponse response = pbRecordFactory.newRecordInstance(
        LocalizerHeartbeatResponse.class);
    Assert.assertEquals(LocalizerHeartbeatResponsePBImpl.class,
                        response.getClass());
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to crete record");
  }
}
Example 17
Source File: TestClientRMService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGetApplicationReport() throws Exception {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);

  ApplicationId appId1 = getApplicationId(1);

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  when(
      mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
          ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);

  ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
      null, mockAclsManager, null, null);
  try {
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    GetApplicationReportRequest request = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    request.setApplicationId(appId1);
    GetApplicationReportResponse response = 
        rmService.getApplicationReport(request);
    ApplicationReport report = response.getApplicationReport();
    ApplicationResourceUsageReport usageReport = 
        report.getApplicationResourceUsageReport();
    Assert.assertEquals(10, usageReport.getMemorySeconds());
    Assert.assertEquals(3, usageReport.getVcoreSeconds());
  } finally {
    rmService.close();
  }
}
Example 18
Source File: TestTaskAttemptListenerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private static TaskAttemptCompletionEvent createTce(int eventId,
    boolean isMap, TaskAttemptCompletionEventStatus status) {
  JobId jid = MRBuilderUtils.newJobId(12345, 1, 1);
  TaskId tid = MRBuilderUtils.newTaskId(jid, 0,
      isMap ? org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP
          : org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  TaskAttemptCompletionEvent tce = recordFactory
      .newRecordInstance(TaskAttemptCompletionEvent.class);
  tce.setEventId(eventId);
  tce.setAttemptId(attemptId);
  tce.setStatus(status);
  return tce;
}
Example 19
Source File: TestNodeHealthService.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testNodeHealthService() throws Exception {
  RecordFactory factory = RecordFactoryProvider.getRecordFactory(null);
  NodeHealthStatus healthStatus =
      factory.newRecordInstance(NodeHealthStatus.class);
  Configuration conf = getConfForNodeHealthScript();
  conf.writeXml(new FileOutputStream(nodeHealthConfigFile));
  conf.addResource(nodeHealthConfigFile.getName());
  writeNodeHealthScriptFile("", true);

  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  NodeHealthScriptRunner nodeHealthScriptRunner =
      spy(NodeManager.getNodeHealthScriptRunner(conf));
  NodeHealthCheckerService nodeHealthChecker = new NodeHealthCheckerService(
  		nodeHealthScriptRunner, dirsHandler);
  nodeHealthChecker.init(conf);

  doReturn(true).when(nodeHealthScriptRunner).isHealthy();
  doReturn("").when(nodeHealthScriptRunner).getHealthReport();
  setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(),
      nodeHealthChecker.getHealthReport(),
      nodeHealthChecker.getLastHealthReportTime());
  LOG.info("Checking initial healthy condition");
  // Check proper report conditions.
  Assert.assertTrue("Node health status reported unhealthy", healthStatus
      .getIsNodeHealthy());
  Assert.assertTrue("Node health status reported unhealthy", healthStatus
      .getHealthReport().equals(nodeHealthChecker.getHealthReport()));

  doReturn(false).when(nodeHealthScriptRunner).isHealthy();   
  // update health status
  setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(),
      nodeHealthChecker.getHealthReport(),
      nodeHealthChecker.getLastHealthReportTime());
  LOG.info("Checking Healthy--->Unhealthy");
  Assert.assertFalse("Node health status reported healthy", healthStatus
      .getIsNodeHealthy());
  Assert.assertTrue("Node health status reported healthy", healthStatus
      .getHealthReport().equals(nodeHealthChecker.getHealthReport()));

  doReturn(true).when(nodeHealthScriptRunner).isHealthy();
  setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(),
      nodeHealthChecker.getHealthReport(),
      nodeHealthChecker.getLastHealthReportTime());
  LOG.info("Checking UnHealthy--->healthy");
  // Check proper report conditions.
  Assert.assertTrue("Node health status reported unhealthy", healthStatus
      .getIsNodeHealthy());
  Assert.assertTrue("Node health status reported unhealthy", healthStatus
      .getHealthReport().equals(nodeHealthChecker.getHealthReport()));

  // Healthy to timeout transition.
  doReturn(false).when(nodeHealthScriptRunner).isHealthy();
  doReturn(NodeHealthScriptRunner.NODE_HEALTH_SCRIPT_TIMED_OUT_MSG)
      .when(nodeHealthScriptRunner).getHealthReport();
  setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(),
      nodeHealthChecker.getHealthReport(),
      nodeHealthChecker.getLastHealthReportTime());
  LOG.info("Checking Healthy--->timeout");
  Assert.assertFalse("Node health status reported healthy even after timeout",
      healthStatus.getIsNodeHealthy());
  Assert.assertTrue("Node script time out message not propogated",
      healthStatus.getHealthReport().equals(
          NodeHealthScriptRunner.NODE_HEALTH_SCRIPT_TIMED_OUT_MSG
          + NodeHealthCheckerService.SEPARATOR
          + nodeHealthChecker.getDiskHandler().getDisksHealthReport(false)));
}
Example 20
Source File: TestNMWebServer.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
  public void testNMWebApp() throws IOException, YarnException {
    Context nmContext = new NodeManager.NMContext(null, null, null, null,
        null, null);
    ResourceView resourceView = new ResourceView() {
      @Override
      public long getVmemAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getPmemAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getVCoresAllocatedForContainers() {
        return 0;
      }
      @Override
      public boolean isVmemCheckEnabled() {
        return true;
      }
      @Override
      public boolean isPmemCheckEnabled() {
        return true;
      }
    };
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
    NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
    healthChecker.init(conf);
    LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();

    WebServer server = new WebServer(nmContext, resourceView,
        new ApplicationACLsManager(conf), dirsHandler);
    server.init(conf);
    server.start();

    // Add an application and the corresponding containers
    RecordFactory recordFactory =
        RecordFactoryProvider.getRecordFactory(conf);
    Dispatcher dispatcher = new AsyncDispatcher();
    String user = "nobody";
    long clusterTimeStamp = 1234;
    ApplicationId appId =
        BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
    Application app = mock(Application.class);
    when(app.getUser()).thenReturn(user);
    when(app.getAppId()).thenReturn(appId);
    nmContext.getApplications().put(appId, app);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
        appId, 1);
    ContainerId container1 =
        BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
    ContainerId container2 =
        BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 1);
    NodeManagerMetrics metrics = mock(NodeManagerMetrics.class);
    NMStateStoreService stateStore = new NMNullStateStoreService();
    for (ContainerId containerId : new ContainerId[] { container1,
        container2}) {
      // TODO: Use builder utils
      ContainerLaunchContext launchContext =
          recordFactory.newRecordInstance(ContainerLaunchContext.class);
      long currentTime = System.currentTimeMillis();
      Token containerToken =
          BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user,
            BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123,
            "password".getBytes(), currentTime);
      Container container =
          new ContainerImpl(nmContext, conf, dispatcher, stateStore, launchContext,
            null, metrics,
            BuilderUtils.newContainerTokenIdentifier(containerToken), null, 0, user, false, null) {

            @Override
            public ContainerState getContainerState() {
              return ContainerState.RUNNING;
            };
          };
      nmContext.getContainers().put(containerId, container);
      //TODO: Gross hack. Fix in code.
      ApplicationId applicationId = 
          containerId.getApplicationAttemptId().getApplicationId();
      nmContext.getApplications().get(applicationId).getContainers()
          .put(containerId, container);
      writeContainerLogs(nmContext, containerId, dirsHandler);

    }
    // TODO: Pull logs and test contents.
//    Thread.sleep(1000000);
  }