org.apache.hadoop.yarn.util.FSDownload Java Examples

The following examples show how to use org.apache.hadoop.yarn.util.FSDownload. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ResourceLocalizationService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * For each of the requested resources for a container, determines the
 * appropriate {@link LocalResourcesTracker} and forwards a 
 * {@link LocalResourceRequest} to that tracker.
 */
private void handleInitContainerResources(
    ContainerLocalizationRequestEvent rsrcReqs) {
  Container c = rsrcReqs.getContainer();
  // create a loading cache for the file statuses
  LoadingCache<Path,Future<FileStatus>> statCache =
      CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig()));
  LocalizerContext ctxt = new LocalizerContext(
      c.getUser(), c.getContainerId(), c.getCredentials(), statCache);
  Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
    rsrcReqs.getRequestedResources();
  for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
       rsrcs.entrySet()) {
    LocalResourcesTracker tracker =
        getLocalResourcesTracker(e.getKey(), c.getUser(),
            c.getContainerId().getApplicationAttemptId()
                .getApplicationId());
    for (LocalResourceRequest req : e.getValue()) {
      tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
    }
  }
}
Example #2
Source File: ResourceLocalizationService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * For each of the requested resources for a container, determines the
 * appropriate {@link LocalResourcesTracker} and forwards a 
 * {@link LocalResourceRequest} to that tracker.
 */
private void handleInitContainerResources(
    ContainerLocalizationRequestEvent rsrcReqs) {
  Container c = rsrcReqs.getContainer();
  // create a loading cache for the file statuses
  LoadingCache<Path,Future<FileStatus>> statCache =
      CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig()));
  LocalizerContext ctxt = new LocalizerContext(
      c.getUser(), c.getContainerId(), c.getCredentials(), statCache);
  Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
    rsrcReqs.getRequestedResources();
  for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
       rsrcs.entrySet()) {
    LocalResourcesTracker tracker =
        getLocalResourcesTracker(e.getKey(), c.getUser(),
            c.getContainerId().getApplicationAttemptId()
                .getApplicationId());
    for (LocalResourceRequest req : e.getValue()) {
      tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
    }
  }
}
Example #3
Source File: ResourceLocalizationService.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public void addResource(LocalizerResourceRequestEvent request) {
  // TODO handle failures, cancellation, requests by other containers
  LocalizedResource rsrc = request.getResource();
  LocalResourceRequest key = rsrc.getRequest();
  LOG.info("Downloading public rsrc:" + key);
  /*
   * Here multiple containers may request the same resource. So we need
   * to start downloading only when
   * 1) ResourceState == DOWNLOADING
   * 2) We are able to acquire non blocking semaphore lock.
   * If not we will skip this resource as either it is getting downloaded
   * or it FAILED / LOCALIZED.
   */

  if (rsrc.tryAcquire()) {
    if (rsrc.getState() == ResourceState.DOWNLOADING) {
      LocalResource resource = request.getResource().getRequest();
      try {
        Path publicRootPath =
            dirsHandler.getLocalPathForWrite("." + Path.SEPARATOR
                + ContainerLocalizer.FILECACHE,
              ContainerLocalizer.getEstimatedSize(resource), true);
        Path publicDirDestPath =
            publicRsrc.getPathForLocalization(key, publicRootPath,
                delService);
        if (!publicDirDestPath.getParent().equals(publicRootPath)) {
          DiskChecker.checkDir(new File(publicDirDestPath.toUri().getPath()));
        }

        // explicitly synchronize pending here to avoid future task
        // completing and being dequeued before pending updated
        synchronized (pending) {
          pending.put(queue.submit(new FSDownload(lfs, null, conf,
              publicDirDestPath, resource, request.getContext().getStatCache())),
              request);
        }
      } catch (IOException e) {
        rsrc.unlock();
        publicRsrc.handle(new ResourceFailedLocalizationEvent(request
          .getResource().getRequest(), e.getMessage()));
        LOG.error("Local path for public localization is not found. "
            + " May be disks failed.", e);
      } catch (IllegalArgumentException ie) {
        rsrc.unlock();
        publicRsrc.handle(new ResourceFailedLocalizationEvent(request
            .getResource().getRequest(), ie.getMessage()));
        LOG.error("Local path for public localization is not found. "
            + " Incorrect path. " + request.getResource().getRequest()
            .getPath(), ie);
      } catch (RejectedExecutionException re) {
        rsrc.unlock();
        publicRsrc.handle(new ResourceFailedLocalizationEvent(request
          .getResource().getRequest(), re.getMessage()));
        LOG.error("Failed to submit rsrc " + rsrc + " for download."
            + " Either queue is full or threadpool is shutdown.", re);
      }
    } else {
      rsrc.unlock();
    }
  }
}
Example #4
Source File: SharedCacheUploader.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@VisibleForTesting
boolean fileIsPublic(final Path remotePath, FileSystem remoteFs,
    FileStatus status) throws IOException {
  return FSDownload.isPublic(remoteFs, remotePath, status, null);
}
Example #5
Source File: ContainerLocalizer.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
Callable<Path> download(Path path, LocalResource rsrc,
    UserGroupInformation ugi) throws IOException {
  DiskChecker.checkDir(new File(path.toUri().getRawPath()));
  return new FSDownload(lfs, ugi, conf, path, rsrc);
}
Example #6
Source File: ResourceLocalizationService.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public void addResource(LocalizerResourceRequestEvent request) {
  // TODO handle failures, cancellation, requests by other containers
  LocalizedResource rsrc = request.getResource();
  LocalResourceRequest key = rsrc.getRequest();
  LOG.info("Downloading public rsrc:" + key);
  /*
   * Here multiple containers may request the same resource. So we need
   * to start downloading only when
   * 1) ResourceState == DOWNLOADING
   * 2) We are able to acquire non blocking semaphore lock.
   * If not we will skip this resource as either it is getting downloaded
   * or it FAILED / LOCALIZED.
   */

  if (rsrc.tryAcquire()) {
    if (rsrc.getState() == ResourceState.DOWNLOADING) {
      LocalResource resource = request.getResource().getRequest();
      try {
        Path publicRootPath =
            dirsHandler.getLocalPathForWrite("." + Path.SEPARATOR
                + ContainerLocalizer.FILECACHE,
              ContainerLocalizer.getEstimatedSize(resource), true);
        Path publicDirDestPath =
            publicRsrc.getPathForLocalization(key, publicRootPath);
        if (!publicDirDestPath.getParent().equals(publicRootPath)) {
          DiskChecker.checkDir(new File(publicDirDestPath.toUri().getPath()));
        }

        // In case this is not a newly initialized nm state, ensure
        // initialized local/log dirs similar to LocalizerRunner
        getInitializedLocalDirs();
        getInitializedLogDirs();

        // explicitly synchronize pending here to avoid future task
        // completing and being dequeued before pending updated
        synchronized (pending) {
          pending.put(queue.submit(new FSDownload(lfs, null, conf,
              publicDirDestPath, resource, request.getContext().getStatCache())),
              request);
        }
      } catch (IOException e) {
        rsrc.unlock();
        publicRsrc.handle(new ResourceFailedLocalizationEvent(request
          .getResource().getRequest(), e.getMessage()));
        LOG.error("Local path for public localization is not found. "
            + " May be disks failed.", e);
      } catch (IllegalArgumentException ie) {
        rsrc.unlock();
        publicRsrc.handle(new ResourceFailedLocalizationEvent(request
            .getResource().getRequest(), ie.getMessage()));
        LOG.error("Local path for public localization is not found. "
            + " Incorrect path. " + request.getResource().getRequest()
            .getPath(), ie);
      } catch (RejectedExecutionException re) {
        rsrc.unlock();
        publicRsrc.handle(new ResourceFailedLocalizationEvent(request
          .getResource().getRequest(), re.getMessage()));
        LOG.error("Failed to submit rsrc " + rsrc + " for download."
            + " Either queue is full or threadpool is shutdown.", re);
      }
    } else {
      rsrc.unlock();
    }
  }
}
Example #7
Source File: SharedCacheUploader.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@VisibleForTesting
boolean fileIsPublic(final Path remotePath, FileSystem remoteFs,
    FileStatus status) throws IOException {
  return FSDownload.isPublic(remoteFs, remotePath, status, null);
}
Example #8
Source File: ContainerLocalizer.java    From big-c with Apache License 2.0 4 votes vote down vote up 
Callable<Path> download(Path path, LocalResource rsrc,
    UserGroupInformation ugi) throws IOException {
  DiskChecker.checkDir(new File(path.toUri().getRawPath()));
  return new FSDownload(lfs, ugi, conf, path, rsrc);
}