Java Code Examples for org.apache.hadoop.yarn.util.ConverterUtils#getPathFromYarnURL()

The following examples show how to use org.apache.hadoop.yarn.util.ConverterUtils#getPathFromYarnURL() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LocalResourceRequest.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Wrap API resource to match against cache of localized resources.
 * @param resource Resource requested by container
 * @throws URISyntaxException If the path is malformed
 */
public LocalResourceRequest(LocalResource resource)
    throws URISyntaxException {
  this(ConverterUtils.getPathFromYarnURL(resource.getResource()),
      resource.getTimestamp(),
      resource.getType(),
      resource.getVisibility(),
      resource.getPattern());
}
Example 2
Source File: SharedCacheUploader.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Checks that the (original) remote file is either owned by the user who
 * started the app or public.
 */
@VisibleForTesting
boolean verifyAccess() throws IOException {
  // if it is in the public cache, it's trivially OK
  if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
    return true;
  }

  final Path remotePath;
  try {
    remotePath = ConverterUtils.getPathFromYarnURL(resource.getResource());
  } catch (URISyntaxException e) {
    throw new IOException("Invalid resource", e);
  }

  // get the file status of the HDFS file
  FileSystem remoteFs = remotePath.getFileSystem(conf);
  FileStatus status = remoteFs.getFileStatus(remotePath);
  // check to see if the file has been modified in any way
  if (status.getModificationTime() != resource.getTimestamp()) {
    LOG.warn("The remote file " + remotePath +
        " has changed since it's localized; will not consider it for upload");
    return false;
  }

  // check for the user ownership
  if (status.getOwner().equals(user)) {
    return true; // the user owns the file
  }
  // check if the file is publicly readable otherwise
  return fileIsPublic(remotePath, remoteFs, status);
}
Example 3
Source File: LocalResourceRequest.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Wrap API resource to match against cache of localized resources.
 * @param resource Resource requested by container
 * @throws URISyntaxException If the path is malformed
 */
public LocalResourceRequest(LocalResource resource)
    throws URISyntaxException {
  this(ConverterUtils.getPathFromYarnURL(resource.getResource()),
      resource.getTimestamp(),
      resource.getType(),
      resource.getVisibility(),
      resource.getPattern());
}
Example 4
Source File: SharedCacheUploader.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Checks that the (original) remote file is either owned by the user who
 * started the app or public.
 */
@VisibleForTesting
boolean verifyAccess() throws IOException {
  // if it is in the public cache, it's trivially OK
  if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
    return true;
  }

  final Path remotePath;
  try {
    remotePath = ConverterUtils.getPathFromYarnURL(resource.getResource());
  } catch (URISyntaxException e) {
    throw new IOException("Invalid resource", e);
  }

  // get the file status of the HDFS file
  FileSystem remoteFs = remotePath.getFileSystem(conf);
  FileStatus status = remoteFs.getFileStatus(remotePath);
  // check to see if the file has been modified in any way
  if (status.getModificationTime() != resource.getTimestamp()) {
    LOG.warn("The remote file " + remotePath +
        " has changed since it's localized; will not consider it for upload");
    return false;
  }

  // check for the user ownership
  if (status.getOwner().equals(user)) {
    return true; // the user owns the file
  }
  // check if the file is publicly readable otherwise
  return fileIsPublic(remotePath, remoteFs, status);
}
Example 5
Source File: DagTypeConverters.java    From tez with Apache License 2.0 5 votes vote down vote up 
public static String convertToDAGPlan(URL resource) {
  Path p;
  try {
    p = ConverterUtils.getPathFromYarnURL(resource);
  } catch (URISyntaxException e) {
    throw new TezUncheckedException("Unable to translate resource: " + resource + " to Path");
  }
  String urlString = p.toString();
  return urlString;
}
Example 6
Source File: DAG.java    From tez with Apache License 2.0 4 votes vote down vote up 
@VisibleForTesting
void verifyLocalResources(Configuration tezConf) {
  for (Vertex v : vertices.values()) {
    for (Map.Entry<String, LocalResource> localResource : v
        .getTaskLocalFiles().entrySet()) {
      String resourceName = localResource.getKey();
      LocalResource resource = localResource.getValue();
      if (commonTaskLocalFiles.containsKey(resourceName)
          && !commonTaskLocalFiles.get(resourceName).equals(resource)) {
        // Different for some reason. Compare size, and then eventually hash
        try {

          LocalResource commonLr = commonTaskLocalFiles.get(resourceName);
          if (resource.getSize() != commonLr.getSize()) {
            throw new IllegalStateException(
                "There is conflicting local resource (size mismatch) (" +
                    resourceName
                    + ") between dag local resource and vertex " +
                    v.getName() + " local resource. "
                    + "\nResource of dag : " +
                    commonTaskLocalFiles.get(resourceName)
                    + "\nResource of vertex: " + resource);
          }

          Path vertexResourcePath =
              ConverterUtils.getPathFromYarnURL(resource.getResource());
          Path commonResourcePath = ConverterUtils.getPathFromYarnURL(
              commonLr.getResource());

          byte[] vertexResourceSha = TezClientUtils
              .getResourceSha(vertexResourcePath.toUri(), tezConf);
          byte[] commonResourceSha = TezClientUtils
              .getResourceSha(commonResourcePath.toUri(), tezConf);

          if (!Arrays.equals(vertexResourceSha, commonResourceSha)) {
            throw new IllegalStateException(
                "There is conflicting local resource (sha mismatch) (" +
                    resourceName
                    + ") between dag local resource and vertex " +
                    v.getName() + " local resource. "
                    + "\nResource of dag : " +
                    commonTaskLocalFiles.get(resourceName)
                    + "\nResource of vertex: " + resource);
          }

        } catch (URISyntaxException | IOException e) {
          throw new RuntimeException(
              "Failed while attempting to validate sha for conflicting resources (" +
                  resourceName
                  + ") between dag local resource and vertex " + v.getName() +
                  " local resource. "
                  + "\nResource of dag : " +
                  commonTaskLocalFiles.get(resourceName)
                  + "\nResource of vertex: " + resource);
        }
      }
    }
  }
}