Java Code Examples for org.apache.hadoop.yarn.api.records.LocalResource#newInstance()

The following examples show how to use org.apache.hadoop.yarn.api.records.LocalResource#newInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TezResourceManager.java    From spork with Apache License 2.0 6 votes vote down vote up 
public Map<String, LocalResource> getTezResources(Set<String> resourceNames) throws Exception {
    Map<String, LocalResource> tezResources = new HashMap<String, LocalResource>();
    for (String resourceName : resourceNames) {
        // The resource name will be symlinked to the resource path in the
        // container's working directory.
        Path resourcePath = resources.get(resourceName);
        FileStatus fstat = remoteFs.getFileStatus(resourcePath);

        LocalResource tezResource = LocalResource.newInstance(
                ConverterUtils.getYarnUrlFromPath(fstat.getPath()),
                LocalResourceType.FILE,
                LocalResourceVisibility.APPLICATION,
                fstat.getLen(),
                fstat.getModificationTime());

        tezResources.put(resourceName, tezResource);
    }
    return tezResources;
}
Example 2
Source File: JstormOnYarn.java    From jstorm with Apache License 2.0 6 votes vote down vote up 
private void addToLocalResources(FileSystem fs, String fileSrcPath,
                                 String fileDstPath, String appId, Map<String, LocalResource> localResources,
                                 String resources) throws IOException {
    String suffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId + JOYConstants.BACKLASH + fileDstPath;
    Path dst =
            new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {
            ostream = FileSystem
                    .create(fs, dst, new FsPermission(JOYConstants.FS_PERMISSION));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc =
            LocalResource.newInstance(
                    ConverterUtils.getYarnUrlFromURI(dst.toUri()),
                    LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
                    scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}
Example 3
Source File: AthenaXYarnClusterDescriptor.java    From AthenaX with Apache License 2.0 5 votes vote down vote up 
private LocalResource toLocalResource(Path path, LocalResourceVisibility visibility) throws IOException {
  FileSystem fs = path.getFileSystem(clusterConf.conf());
  FileStatus stat = fs.getFileStatus(path);
  return LocalResource.newInstance(
          ConverterUtils.getYarnUrlFromPath(path),
          LocalResourceType.FILE,
          visibility,
          stat.getLen(), stat.getModificationTime()
  );
}
Example 4
Source File: Utils.java    From TensorFlowOnYARN with Apache License 2.0 5 votes vote down vote up 
private static void addToLocalResources(FileSystem fs, String key, Path dst,
    Map<String, LocalResource> localResources) throws IOException {
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource resource =
      LocalResource.newInstance(
          URL.fromURI(dst.toUri()),
          LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
          scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(key, resource);
}
Example 5
Source File: DagTypeConverters.java    From tez with Apache License 2.0 5 votes vote down vote up 
public static LocalResource convertPlanLocalResourceToLocalResource(
    PlanLocalResource plr) {
  return LocalResource.newInstance(
      ConverterUtils.getYarnUrlFromPath(new Path(plr.getUri())),
      DagTypeConverters.convertFromDAGPlan(plr.getType()),
      DagTypeConverters.convertFromDAGPlan(plr.getVisibility()),
      plr.getSize(), plr.getTimeStamp(),
      plr.hasPattern() ? plr.getPattern() : null);
}
Example 6
Source File: TaskAttemptImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
    resourceSize, resourceModificationTime);
}
Example 7
Source File: Client.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void addToLocalResources(FileSystem fs, String fileSrcPath,
    String fileDstPath, String appId, Map<String, LocalResource> localResources,
    String resources) throws IOException {
  String suffix =
      appName + "/" + appId + "/" + fileDstPath;
  Path dst =
      new Path(fs.getHomeDirectory(), suffix);
  if (fileSrcPath == null) {
    FSDataOutputStream ostream = null;
    try {
      ostream = FileSystem
          .create(fs, dst, new FsPermission((short) 0710));
      ostream.writeUTF(resources);
    } finally {
      IOUtils.closeQuietly(ostream);
    }
  } else {
    fs.copyFromLocalFile(new Path(fileSrcPath), dst);
  }
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource scRsrc =
      LocalResource.newInstance(
          ConverterUtils.getYarnUrlFromURI(dst.toUri()),
          LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
          scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(fileDstPath, scRsrc);
}
Example 8
Source File: TaskAttemptImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
    resourceSize, resourceModificationTime);
}
Example 9
Source File: Client.java    From metron with Apache License 2.0 5 votes vote down vote up 
private Path addToLocalResources(FileSystem fs, String fileSrcPath,
                                 String fileDstPath, String appId, Map<String, LocalResource> localResources,
                                 String resources) throws IOException {
  String suffix =
          appName + "/" + appId + "/" + fileDstPath;
  Path dst =
          new Path(fs.getHomeDirectory(), suffix);
  if (fileSrcPath == null) {
    FSDataOutputStream ostream = null;
    try {
      ostream = FileSystem
              .create(fs, dst, new FsPermission((short) 0710));
      ostream.writeUTF(resources);
    } finally {
      IOUtils.closeQuietly(ostream);
    }
  } else {
    fs.copyFromLocalFile(new Path(fileSrcPath), dst);
  }
  fs.setPermission(dst, new FsPermission((short)0755));
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource scRsrc =
          LocalResource.newInstance(
                  ConverterUtils.getYarnUrlFromURI(dst.toUri()),
                  LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
                  scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(fileDstPath, scRsrc);
  return dst;
}
Example 10
Source File: TestMRRJobsDAGApi.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}
Example 11
Source File: AMContainerHelpers.java    From tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
public static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}
Example 12
Source File: DagTypeConverters.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
public static LocalResource convertPlanLocalResourceToLocalResource(
    PlanLocalResource plr) {
  return LocalResource.newInstance(
    ConverterUtils.getYarnUrlFromPath(new Path(plr.getUri())),
    DagTypeConverters.convertFromDAGPlan(plr.getType()),
    DagTypeConverters.convertFromDAGPlan(plr.getVisibility()),
    plr.getSize(), plr.getTimeStamp(),
    plr.hasPattern() ? plr.getPattern() : null);
}
Example 13
Source File: TestMRRJobsDAGApi.java    From tez with Apache License 2.0 5 votes vote down vote up 
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}
Example 14
Source File: TestAMContainer.java    From tez with Apache License 2.0 4 votes vote down vote up 
private LocalResource createLocalResource(String name) {
  LocalResource lr = LocalResource.newInstance(URL.newInstance(null, "localhost", 2321, name),
      LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 1, 1000000);
  return lr;
}
Example 15
Source File: LaunchContainerRunnable.java    From Bats with Apache License 2.0 4 votes vote down vote up 
public static void addFileToLocalResources(final String name, final FileStatus fileStatus, final LocalResourceType type, final Map<String, LocalResource> localResources)
{
  final LocalResource localResource = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fileStatus.getPath()),
      type, LocalResourceVisibility.APPLICATION, fileStatus.getLen(), fileStatus.getModificationTime());
  localResources.put(name, localResource);
}
Example 16
Source File: TestDAGClientAMProtocolBlockingPBServerImpl.java    From tez with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 100000)
@SuppressWarnings("unchecked")
public void testSubmitDagInSessionWithLargeDagPlan() throws Exception {
  int maxIPCMsgSize = 1024;
  String dagPlanName = "dagplan-name";
  File requestFile = tmpFolder.newFile("request-file");
  TezConfiguration conf = new TezConfiguration();
  conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize);

  // Check with 70 MB (64 MB is CodedInputStream's default limit in earlier versions of protobuf)
  byte[] randomBytes = new byte[70 << 20];
  (new Random()).nextBytes(randomBytes);
  UserPayload payload = UserPayload.create(ByteBuffer.wrap(randomBytes));
  Vertex vertex = Vertex.create("V", ProcessorDescriptor.create("P").setUserPayload(payload), 1);
  DAGPlan dagPlan = DAG.create(dagPlanName).addVertex(vertex).createDag(conf, null, null, null, false);

  String lrName = "localResource";
  String scheme = "file";
  String host = "localhost";
  int port = 80;
  String path = "/test";
  URL lrURL = URL.newInstance(scheme, host, port, path);
  LocalResource localResource = LocalResource.newInstance(lrURL, LocalResourceType.FILE,
      LocalResourceVisibility.PUBLIC, 1, 1);
  Map<String, LocalResource> localResources = new HashMap<>();
  localResources.put(lrName, localResource);

  SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan)
      .setAdditionalAmResources(DagTypeConverters.convertFromLocalResources(localResources));
  try (FileOutputStream fileOutputStream = new FileOutputStream(requestFile)) {
    requestBuilder.build().writeTo(fileOutputStream);
  }

  DAGClientHandler dagClientHandler = mock(DAGClientHandler.class);
  ACLManager aclManager = mock(ACLManager.class);
  DAGClientAMProtocolBlockingPBServerImpl serverImpl = spy(new DAGClientAMProtocolBlockingPBServerImpl(
      dagClientHandler, FileSystem.get(conf)));
  when(dagClientHandler.getACLManager()).thenReturn(aclManager);
  when(dagClientHandler.submitDAG((DAGPlan)any(), (Map<String, LocalResource>)any())).thenReturn("dag-id");
  when(aclManager.checkAMModifyAccess((UserGroupInformation) any())).thenReturn(true);

  requestBuilder.clear().setSerializedRequestPath(requestFile.getAbsolutePath());
  serverImpl.submitDAG(null, requestBuilder.build());

  ArgumentCaptor<DAGPlan> dagPlanCaptor = ArgumentCaptor.forClass(DAGPlan.class);
  verify(dagClientHandler).submitDAG(dagPlanCaptor.capture(), localResourcesCaptor.capture());
  dagPlan = dagPlanCaptor.getValue();
  localResources = localResourcesCaptor.getValue();

  assertEquals(dagPlan.getName(), dagPlanName);
  assertEquals(dagPlan.getVertexCount(), 1);
  assertTrue(dagPlan.getSerializedSize() > maxIPCMsgSize);
  assertArrayEquals(randomBytes, dagPlan.getVertex(0).getProcessorDescriptor().getTezUserPayload().getUserPayload().
      toByteArray());
  assertEquals(localResources.size(), 1);
  assertTrue(localResources.containsKey(lrName));
  localResource = localResources.get(lrName);
  assertEquals(localResource.getType(), LocalResourceType.FILE);
  assertEquals(localResource.getVisibility(), LocalResourceVisibility.PUBLIC);
  lrURL = localResource.getResource();
  assertEquals(lrURL.getScheme(), scheme);
  assertEquals(lrURL.getHost(), host);
  assertEquals(lrURL.getPort(), port);
  assertEquals(lrURL.getFile(), path);
}
Example 17
Source File: TestMRRJobsDAGApi.java    From tez with Apache License 2.0 4 votes vote down vote up 
private LocalResource createLrObjFromPath(Path filePath) {
  return LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(filePath),
      LocalResourceType.FILE, LocalResourceVisibility.PRIVATE, 0, 0);
}
Example 18
Source File: TestAMContainer.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up 
private LocalResource createLocalResource(String name) {
  LocalResource lr = LocalResource.newInstance(URL.newInstance(null, "localhost", 2321, name),
      LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 1, 1000000);
  return lr;
}
Example 19
Source File: TestMRRJobsDAGApi.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up 
private LocalResource createLrObjFromPath(Path filePath) {
  return LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(filePath),
      LocalResourceType.FILE, LocalResourceVisibility.PRIVATE, 0, 0);
}
Example 20
Source File: LaunchContainerRunnable.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up 
public static void addFileToLocalResources(final String name, final FileStatus fileStatus, final LocalResourceType type, final Map<String, LocalResource> localResources)
{
  final LocalResource localResource = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fileStatus.getPath()),
      type, LocalResourceVisibility.APPLICATION, fileStatus.getLen(), fileStatus.getModificationTime());
  localResources.put(name, localResource);
}