org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider Java Examples

The following examples show how to use org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestAppManager.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
Example #2
Source File: SharedCacheUploader.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * @param resource the local resource that contains the original remote path
 * @param localPath the path in the local filesystem where the resource is
 * localized
 * @param fs the filesystem of the shared cache
 * @param localFs the local filesystem
 */
public SharedCacheUploader(LocalResource resource, Path localPath,
    String user, Configuration conf, SCMUploaderProtocol scmClient,
    FileSystem fs, FileSystem localFs) {
  this.resource = resource;
  this.localPath = localPath;
  this.user = user;
  this.conf = conf;
  this.scmClient = scmClient;
  this.fs = fs;
  this.sharedCacheRootDir =
      conf.get(YarnConfiguration.SHARED_CACHE_ROOT,
          YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT);
  this.nestedLevel = SharedCacheUtil.getCacheDepth(conf);
  this.checksum = SharedCacheChecksumFactory.getChecksum(conf);
  this.localFs = localFs;
  this.recordFactory = RecordFactoryProvider.getRecordFactory(null);
}
Example #3
Source File: MockContainer.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public MockContainer(ApplicationAttemptId appAttemptId,
    Dispatcher dispatcher, Configuration conf, String user,
    ApplicationId appId, int uniqId) throws IOException{

  this.user = user;
  this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
  this.id = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId,
      uniqId);
  this.launchContext = recordFactory
      .newRecordInstance(ContainerLaunchContext.class);
  long currentTime = System.currentTimeMillis();
  this.containerTokenIdentifier =
      BuilderUtils.newContainerTokenIdentifier(BuilderUtils
        .newContainerToken(id, "127.0.0.1", 1234, user,
          BuilderUtils.newResource(1024, 1), currentTime + 10000, 123,
          "password".getBytes(), currentTime));
  this.state = ContainerState.NEW;
}
Example #4
Source File: MiniYARNClusterSplice.java    From spliceengine with GNU Affero General Public License v3.0 6 votes vote down vote up 
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
                                                    Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
    return new NodeStatusUpdaterImpl(context, dispatcher,
                                     healthChecker, metrics) {
        @Override
        protected ResourceTracker getRMClient() {
            final ResourceTrackerService rt =
                getResourceManager().getResourceTrackerService();
            final RecordFactory recordFactory =
                RecordFactoryProvider.getRecordFactory(null);

            // For in-process communication without RPC
            return Utils.getResourceTracker(rt);
        }

        @Override
        protected void stopRMProxy() { }
     };
}
Example #5
Source File: TestTezLocalCacheManager.java    From tez with Apache License 2.0 6 votes vote down vote up 
private static LocalResource createFile(String content) throws IOException {
  FileContext fs = FileContext.getLocalFSFileContext();

  java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt");
  File temp = tempFile.toFile();
  temp.deleteOnExit();
  Path p = new Path("file:///" + tempFile.toAbsolutePath().toString());

  Files.write(tempFile, content.getBytes());

  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p);
  ret.setResource(yarnUrlFromPath);
  ret.setSize(content.getBytes().length);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(LocalResourceVisibility.PRIVATE);
  ret.setTimestamp(fs.getFileStatus(p).getModificationTime());
  return ret;
}
Example #6
Source File: TestAppManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
Example #7
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNonExistingApplicationReport() throws YarnException {
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getRMApps()).thenReturn(
      new ConcurrentHashMap<ApplicationId, RMApp>());
  ClientRMService rmService = new ClientRMService(rmContext, null, null,
      null, null, null);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationReportRequest request = recordFactory
      .newRecordInstance(GetApplicationReportRequest.class);
  request.setApplicationId(ApplicationId.newInstance(0, 0));
  try {
    rmService.getApplicationReport(request);
    Assert.fail();
  } catch (ApplicationNotFoundException ex) {
    Assert.assertEquals(ex.getMessage(),
        "Application with id '" + request.getApplicationId()
            + "' doesn't exist in RM.");
  }
}
Example #8
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetApplicationAttemptReport() throws YarnException,
    IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptReportRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationAttemptId(attemptId);

  try {
    GetApplicationAttemptReportResponse response = rmService
        .getApplicationAttemptReport(request);
    Assert.assertEquals(attemptId, response.getApplicationAttemptReport()
        .getApplicationAttemptId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
Example #9
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetApplicationAttempts() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptsRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptsRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationId(ApplicationId.newInstance(123456, 1));

  try {
    GetApplicationAttemptsResponse response = rmService
        .getApplicationAttempts(request);
    Assert.assertEquals(1, response.getApplicationAttemptList().size());
    Assert.assertEquals(attemptId, response.getApplicationAttemptList()
        .get(0).getApplicationAttemptId());

  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
Example #10
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetContainers() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainersRequest request = recordFactory
      .newRecordInstance(GetContainersRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setApplicationAttemptId(attemptId);
  try {
    GetContainersResponse response = rmService.getContainers(request);
    Assert.assertEquals(containerId, response.getContainerList().get(0)
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
Example #11
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetContainers() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainersRequest request = recordFactory
      .newRecordInstance(GetContainersRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setApplicationAttemptId(attemptId);
  try {
    GetContainersResponse response = rmService.getContainers(request);
    Assert.assertEquals(containerId, response.getContainerList().get(0)
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
Example #12
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNonExistingApplicationReport() throws YarnException {
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getRMApps()).thenReturn(
      new ConcurrentHashMap<ApplicationId, RMApp>());
  ClientRMService rmService = new ClientRMService(rmContext, null, null,
      null, null, null);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationReportRequest request = recordFactory
      .newRecordInstance(GetApplicationReportRequest.class);
  request.setApplicationId(ApplicationId.newInstance(0, 0));
  try {
    rmService.getApplicationReport(request);
    Assert.fail();
  } catch (ApplicationNotFoundException ex) {
    Assert.assertEquals(ex.getMessage(),
        "Application with id '" + request.getApplicationId()
            + "' doesn't exist in RM.");
  }
}
Example #13
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetApplicationAttemptReport() throws YarnException,
    IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptReportRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationAttemptId(attemptId);

  try {
    GetApplicationAttemptReportResponse response = rmService
        .getApplicationAttemptReport(request);
    Assert.assertEquals(attemptId, response.getApplicationAttemptReport()
        .getApplicationAttemptId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
Example #14
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetApplicationAttempts() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptsRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptsRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationId(ApplicationId.newInstance(123456, 1));

  try {
    GetApplicationAttemptsResponse response = rmService
        .getApplicationAttempts(request);
    Assert.assertEquals(1, response.getApplicationAttemptList().size());
    Assert.assertEquals(attemptId, response.getApplicationAttemptList()
        .get(0).getApplicationAttemptId());

  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
Example #15
Source File: MockContainer.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public MockContainer(ApplicationAttemptId appAttemptId,
    Dispatcher dispatcher, Configuration conf, String user,
    ApplicationId appId, int uniqId) throws IOException{

  this.user = user;
  this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
  this.id = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId,
      uniqId);
  this.launchContext = recordFactory
      .newRecordInstance(ContainerLaunchContext.class);
  long currentTime = System.currentTimeMillis();
  this.containerTokenIdentifier =
      BuilderUtils.newContainerTokenIdentifier(BuilderUtils
        .newContainerToken(id, "127.0.0.1", 1234, user,
          BuilderUtils.newResource(1024, 1), currentTime + 10000, 123,
          "password".getBytes(), currentTime));
  this.state = ContainerState.NEW;
}
Example #16
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetContainerReport() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainerReportRequest request = recordFactory
      .newRecordInstance(GetContainerReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setContainerId(containerId);

  try {
    GetContainerReportResponse response = rmService
        .getContainerReport(request);
    Assert.assertEquals(containerId, response.getContainerReport()
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
Example #17
Source File: SharedCacheUploader.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * @param resource the local resource that contains the original remote path
 * @param localPath the path in the local filesystem where the resource is
 * localized
 * @param fs the filesystem of the shared cache
 * @param localFs the local filesystem
 */
public SharedCacheUploader(LocalResource resource, Path localPath,
    String user, Configuration conf, SCMUploaderProtocol scmClient,
    FileSystem fs, FileSystem localFs) {
  this.resource = resource;
  this.localPath = localPath;
  this.user = user;
  this.conf = conf;
  this.scmClient = scmClient;
  this.fs = fs;
  this.sharedCacheRootDir =
      conf.get(YarnConfiguration.SHARED_CACHE_ROOT,
          YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT);
  this.nestedLevel = SharedCacheUtil.getCacheDepth(conf);
  this.checksum = SharedCacheChecksumFactory.getChecksum(conf);
  this.localFs = localFs;
  this.recordFactory = RecordFactoryProvider.getRecordFactory(null);
}
Example #18
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetContainerReport() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainerReportRequest request = recordFactory
      .newRecordInstance(GetContainerReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setContainerId(containerId);

  try {
    GetContainerReportResponse response = rmService
        .getContainerReport(request);
    Assert.assertEquals(containerId, response.getContainerReport()
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
Example #19
Source File: ConverterUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static URL getYarnUrlFromURI(URI uri) {
  URL url = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(URL.class);
  if (uri.getHost() != null) {
    url.setHost(uri.getHost());
  }
  if (uri.getUserInfo() != null) {
    url.setUserInfo(uri.getUserInfo());
  }
  url.setPort(uri.getPort());
  url.setScheme(uri.getScheme());
  url.setFile(uri.getPath());
  return url;
}
Example #20
Source File: ConverterUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static URL getYarnUrlFromURI(URI uri) {
  URL url = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(URL.class);
  if (uri.getHost() != null) {
    url.setHost(uri.getHost());
  }
  if (uri.getUserInfo() != null) {
    url.setUserInfo(uri.getUserInfo());
  }
  url.setPort(uri.getPort());
  url.setScheme(uri.getScheme());
  url.setFile(uri.getPath());
  return url;
}
Example #21
Source File: ResourceLocalizationService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public void serviceInit(Configuration conf) throws Exception {
  this.validateConf(conf);
  this.publicRsrc = new LocalResourcesTrackerImpl(null, null, dispatcher,
      true, conf, stateStore);
  this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);

  try {
    lfs = getLocalFileContext(conf);
    lfs.setUMask(new FsPermission((short) FsPermission.DEFAULT_UMASK));

    if (!stateStore.canRecover()|| stateStore.isNewlyCreated()) {
      cleanUpLocalDirs(lfs, delService);
      initializeLocalDirs(lfs);
      initializeLogDirs(lfs);
    }
  } catch (Exception e) {
    throw new YarnRuntimeException(
      "Failed to initialize LocalizationService", e);
  }

  cacheTargetSize =
    conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_TARGET_SIZE_MB, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB) << 20;
  cacheCleanupPeriod =
    conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS);
  localizationServerAddress = conf.getSocketAddr(
      YarnConfiguration.NM_BIND_HOST,
      YarnConfiguration.NM_LOCALIZER_ADDRESS,
      YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
      YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);

  localizerTracker = createLocalizerTracker(conf);
  addService(localizerTracker);
  dispatcher.register(LocalizerEventType.class, localizerTracker);
  super.serviceInit(conf);
}
Example #22
Source File: DefaultContainerExecutor.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public void startLocalizer(LocalizerStartContext ctx)
    throws IOException, InterruptedException {
  Path nmPrivateContainerTokensPath = ctx.getNmPrivateContainerTokens();
  InetSocketAddress nmAddr = ctx.getNmAddr();
  String user = ctx.getUser();
  String appId = ctx.getAppId();
  String locId = ctx.getLocId();
  LocalDirsHandlerService dirsHandler = ctx.getDirsHandler();

  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  
  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);


  FileContext localizerFc = FileContext.getFileContext(
      lfs.getDefaultFileSystem(), getConf());
  localizerFc.setUMask(lfs.getUMask());
  localizerFc.setWorkingDirectory(appStorageDir);
  LOG.info("Localizer CWD set to " + appStorageDir + " = " 
      + localizerFc.getWorkingDirectory());
  ContainerLocalizer localizer =
      new ContainerLocalizer(localizerFc, user, appId, locId, 
          getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf()));
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
Example #23
Source File: DockerContainerExecutor.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
                                        InetSocketAddress nmAddr, String user, String appId, String locId,
                                        LocalDirsHandlerService dirsHandler)
  throws IOException, InterruptedException {

  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();

  ContainerLocalizer localizer =
    new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
      RecordFactoryProvider.getRecordFactory(getConf()));

  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  //LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
  lfs.setWorkingDirectory(appStorageDir);
  //LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory());
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
Example #24
Source File: DefaultContainerExecutor.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public void startLocalizer(Path nmPrivateContainerTokensPath,
    InetSocketAddress nmAddr, String user, String appId, String locId,
    LocalDirsHandlerService dirsHandler)
    throws IOException, InterruptedException {

  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  
  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);


  FileContext localizerFc = FileContext.getFileContext(
      lfs.getDefaultFileSystem(), getConf());
  localizerFc.setUMask(lfs.getUMask());
  localizerFc.setWorkingDirectory(appStorageDir);
  LOG.info("Localizer CWD set to " + appStorageDir + " = " 
      + localizerFc.getWorkingDirectory());
  ContainerLocalizer localizer =
      new ContainerLocalizer(localizerFc, user, appId, locId, 
          getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf()));
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
Example #25
Source File: TestLocalResource.java    From big-c with Apache License 2.0 5 votes vote down vote up 
static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path p, long size,
    long timestamp, LocalResourceType type, LocalResourceVisibility state, String pattern)
    throws URISyntaxException {
  org.apache.hadoop.yarn.api.records.LocalResource ret = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(org.apache.hadoop.yarn.api.records.LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromURI(p.toUri()));
  ret.setSize(size);
  ret.setTimestamp(timestamp);
  ret.setType(type);
  ret.setVisibility(state);
  ret.setPattern(pattern);
  return ret;
}
Example #26
Source File: NodeManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static org.apache.hadoop.yarn.server.api.records.NodeStatus 
createNodeStatus(NodeId nodeId, List<ContainerStatus> containers) {
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = 
      recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class);
  nodeStatus.setNodeId(nodeId);
  nodeStatus.setContainersStatuses(containers);
  NodeHealthStatus nodeHealthStatus = 
    recordFactory.newRecordInstance(NodeHealthStatus.class);
  nodeHealthStatus.setIsNodeHealthy(true);
  nodeStatus.setNodeHealthStatus(nodeHealthStatus);
  return nodeStatus;
}
Example #27
Source File: TestClientRMService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGetApplicationReport() throws Exception {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);

  ApplicationId appId1 = getApplicationId(1);

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  when(
      mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
          ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);

  ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
      null, mockAclsManager, null, null);
  try {
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    GetApplicationReportRequest request = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    request.setApplicationId(appId1);
    GetApplicationReportResponse response = 
        rmService.getApplicationReport(request);
    ApplicationReport report = response.getApplicationReport();
    ApplicationResourceUsageReport usageReport = 
        report.getApplicationResourceUsageReport();
    Assert.assertEquals(10, usageReport.getMemorySeconds());
    Assert.assertEquals(3, usageReport.getVcoreSeconds());
  } finally {
    rmService.close();
  }
}
Example #28
Source File: DockerContainerExecutor.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public synchronized void startLocalizer(LocalizerStartContext ctx)
  throws IOException, InterruptedException {
  Path nmPrivateContainerTokensPath = ctx.getNmPrivateContainerTokens();
  InetSocketAddress nmAddr = ctx.getNmAddr();
  String user = ctx.getUser();
  String appId = ctx.getAppId();
  String locId = ctx.getLocId();
  LocalDirsHandlerService dirsHandler = ctx.getDirsHandler();
  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();

  ContainerLocalizer localizer =
    new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
      RecordFactoryProvider.getRecordFactory(getConf()));

  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
  lfs.setWorkingDirectory(appStorageDir);
  LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory());
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
Example #29
Source File: MockApp.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public MockApp(String user, long clusterTimeStamp, int uniqId) {
  super();
  this.user = user;
  // Add an application and the corresponding containers
  RecordFactory recordFactory = RecordFactoryProvider
      .getRecordFactory(new Configuration());
  this.appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp,
      uniqId);
  appState = ApplicationState.NEW;
}
Example #30
Source File: TokenUtils.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up 
private static Token<?> getDelegationTokenFromHS(HSClientProtocol hsProxy, Configuration conf) throws IOException {
  GetDelegationTokenRequest request =
      RecordFactoryProvider.getRecordFactory(null).newRecordInstance(GetDelegationTokenRequest.class);
  request.setRenewer(Master.getMasterPrincipal(conf));
  org.apache.hadoop.yarn.api.records.Token mrDelegationToken;
  mrDelegationToken = hsProxy.getDelegationToken(request).getDelegationToken();
  return ConverterUtils.convertFromYarn(mrDelegationToken, hsProxy.getConnectAddress());
}