Java Code Examples for org.apache.hadoop.yarn.api.records.LocalResourceVisibility#PUBLIC

@Test
public void testFromString() throws Exception {
	final YarnLocalResourceDescriptor localResourceDesc = new YarnLocalResourceDescriptor(
		key,
		path,
		size,
		ts,
		LocalResourceVisibility.PUBLIC);

	final String desc = localResourceDesc.toString();
	YarnLocalResourceDescriptor newLocalResourceDesc = YarnLocalResourceDescriptor.fromString(desc);
	assertThat(newLocalResourceDesc.getResourceKey(), is(key));
	assertThat(newLocalResourceDesc.getPath(), is(path));
	assertThat(newLocalResourceDesc.getSize(), is(size));
	assertThat(newLocalResourceDesc.getModificationTime(), is(ts));
	assertThat(newLocalResourceDesc.getVisibility(), is(LocalResourceVisibility.PUBLIC));
}
 

LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
    long nRsrcs, long timestamp, long tsstep) {
  Configuration conf = new Configuration();
  ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
    new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
  LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
    null, trackerResources, false, conf, new NMNullStateStoreService()));
  for (int i = 0; i < nRsrcs; ++i) {
    final LocalResourceRequest req = new LocalResourceRequest(
        new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
        LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
    final long ts = timestamp + i * tsstep;
    final Path p = new Path("file:///local/" + user + "/rsrc" + i);
    LocalizedResource rsrc = new LocalizedResource(req, null) {
      @Override public int getRefCount() { return 0; }
      @Override public long getSize() { return rsrcSize; }
      @Override public Path getLocalPath() { return p; }
      @Override public long getTimestamp() { return ts; }
      @Override
      public ResourceState getState() { return ResourceState.LOCALIZED; }
    };
    trackerResources.put(req, rsrc);
  }
  return ret;
}
 

LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
    long nRsrcs, long timestamp, long tsstep) {
  Configuration conf = new Configuration();
  ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
    new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
  LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
    null, trackerResources, false, conf, new NMNullStateStoreService(),null));
  for (int i = 0; i < nRsrcs; ++i) {
    final LocalResourceRequest req = new LocalResourceRequest(
        new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
        LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
    final long ts = timestamp + i * tsstep;
    final Path p = new Path("file:///local/" + user + "/rsrc" + i);
    LocalizedResource rsrc = new LocalizedResource(req, null) {
      @Override public int getRefCount() { return 0; }
      @Override public long getSize() { return rsrcSize; }
      @Override public Path getLocalPath() { return p; }
      @Override public long getTimestamp() { return ts; }
      @Override
      public ResourceState getState() { return ResourceState.LOCALIZED; }
    };
    trackerResources.put(req, rsrc);
  }
  return ret;
}
 

private Path copy(Path sCopy, Path dstdir) throws IOException {
  FileSystem sourceFs = sCopy.getFileSystem(conf);
  Path dCopy = new Path(dstdir, "tmp_"+sCopy.getName());
  FileStatus sStat = sourceFs.getFileStatus(sCopy);
  if (sStat.getModificationTime() != resource.getTimestamp()) {
    throw new IOException("Resource " + sCopy +
        " changed on src filesystem (expected " + resource.getTimestamp() +
        ", was " + sStat.getModificationTime());
  }
  if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
    if (!isPublic(sourceFs, sCopy, sStat, statCache)) {
      throw new IOException("Resource " + sCopy +
          " is not publicly accessable and as such cannot be part of the" +
          " public cache.");
    }
  }

  FileUtil.copy(sourceFs, sStat, FileSystem.getLocal(conf), dCopy, false,
      true, conf);
  return dCopy;
}
 

private Path copy(Path sCopy, Path dstdir) throws IOException {
  FileSystem sourceFs = sCopy.getFileSystem(conf);
  Path dCopy = new Path(dstdir, "tmp_"+sCopy.getName());
  FileStatus sStat = sourceFs.getFileStatus(sCopy);
  if (sStat.getModificationTime() != resource.getTimestamp()) {
    throw new IOException("Resource " + sCopy +
        " changed on src filesystem (expected " + resource.getTimestamp() +
        ", was " + sStat.getModificationTime());
  }
  if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
    if (!isPublic(sourceFs, sCopy, sStat, statCache)) {
      throw new IOException("Resource " + sCopy +
          " is not publicly accessable and as such cannot be part of the" +
          " public cache.");
    }
  }

  FileUtil.copy(sourceFs, sStat, FileSystem.getLocal(conf), dCopy, false,
      true, conf);
  return dCopy;
}
 

public static LocalResourceVisibility convertFromDAGPlan(PlanLocalResourceVisibility visibility){
  switch(visibility){
    case PUBLIC : return LocalResourceVisibility.PUBLIC;
    case PRIVATE : return LocalResourceVisibility.PRIVATE;
    case APPLICATION : return LocalResourceVisibility.APPLICATION;
    default : throw new RuntimeException("unknown 'visibility': " + visibility);
  }
}
 

private void verifyPermsRecursively(FileSystem fs,
    FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  FileStatus status = files.getFileStatus(p);
  if (status.isDirectory()) {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_DIR_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_DIR_PERMS.toShort());
    }
    if (!status.isSymlink()) {
      FileStatus[] statuses = fs.listStatus(p);
      for (FileStatus stat : statuses) {
        verifyPermsRecursively(fs, files, stat.getPath(), vis);
      }
    }
  }
  else {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_FILE_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_FILE_PERMS.toShort());
    }
  }      
}
 

/**
 * Checks that the (original) remote file is either owned by the user who
 * started the app or public.
 */
@VisibleForTesting
boolean verifyAccess() throws IOException {
  // if it is in the public cache, it's trivially OK
  if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
    return true;
  }

  final Path remotePath;
  try {
    remotePath = ConverterUtils.getPathFromYarnURL(resource.getResource());
  } catch (URISyntaxException e) {
    throw new IOException("Invalid resource", e);
  }

  // get the file status of the HDFS file
  FileSystem remoteFs = remotePath.getFileSystem(conf);
  FileStatus status = remoteFs.getFileStatus(remotePath);
  // check to see if the file has been modified in any way
  if (status.getModificationTime() != resource.getTimestamp()) {
    LOG.warn("The remote file " + remotePath +
        " has changed since it's localized; will not consider it for upload");
    return false;
  }

  // check for the user ownership
  if (status.getOwner().equals(user)) {
    return true; // the user owns the file
  }
  // check if the file is publicly readable otherwise
  return fileIsPublic(remotePath, remoteFs, status);
}
 

public static LocalResourceVisibility convertFromDAGPlan(PlanLocalResourceVisibility visibility){
  switch(visibility){
    case PUBLIC : return LocalResourceVisibility.PUBLIC;
    case PRIVATE : return LocalResourceVisibility.PRIVATE;
    case APPLICATION : return LocalResourceVisibility.APPLICATION;
    default : throw new RuntimeException("unknown 'visibility': " + visibility);
  }
}
 

private void verifyPermsRecursively(FileSystem fs,
    FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  FileStatus status = files.getFileStatus(p);
  if (status.isDirectory()) {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_DIR_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_DIR_PERMS.toShort());
    }
    if (!status.isSymlink()) {
      FileStatus[] statuses = fs.listStatus(p);
      for (FileStatus stat : statuses) {
        verifyPermsRecursively(fs, files, stat.getPath(), vis);
      }
    }
  }
  else {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_FILE_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_FILE_PERMS.toShort());
    }
  }      
}
 

/**
 * Checks that the (original) remote file is either owned by the user who
 * started the app or public.
 */
@VisibleForTesting
boolean verifyAccess() throws IOException {
  // if it is in the public cache, it's trivially OK
  if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
    return true;
  }

  final Path remotePath;
  try {
    remotePath = ConverterUtils.getPathFromYarnURL(resource.getResource());
  } catch (URISyntaxException e) {
    throw new IOException("Invalid resource", e);
  }

  // get the file status of the HDFS file
  FileSystem remoteFs = remotePath.getFileSystem(conf);
  FileStatus status = remoteFs.getFileStatus(remotePath);
  // check to see if the file has been modified in any way
  if (status.getModificationTime() != resource.getTimestamp()) {
    LOG.warn("The remote file " + remotePath +
        " has changed since it's localized; will not consider it for upload");
    return false;
  }

  // check for the user ownership
  if (status.getOwner().equals(user)) {
    return true; // the user owns the file
  }
  // check if the file is publicly readable otherwise
  return fileIsPublic(remotePath, remoteFs, status);
}
 

/**
 * Recursively change permissions of all files/dirs on path based 
 * on resource visibility.
 * Change to 755 or 700 for dirs, 555 or 500 for files.
 * @param fs FileSystem
 * @param path Path to modify perms for
 * @throws IOException
 * @throws InterruptedException 
 */
private void changePermissions(FileSystem fs, final Path path)
    throws IOException, InterruptedException {
  File f = new File(path.toUri());
  if (FileUtils.isSymlink(f)) {
    // avoid following symlinks when changing permissions
    return;
  }
  boolean isDir = f.isDirectory();
  FsPermission perm = cachePerms;
  // set public perms as 755 or 555 based on dir or file
  if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
    perm = isDir ? PUBLIC_DIR_PERMS : PUBLIC_FILE_PERMS;
  }
  // set private perms as 700 or 500
  else {
    // PRIVATE:
    // APPLICATION:
    perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
  }
  LOG.debug("Changing permissions for path " + path
      + " to perm " + perm);
  final FsPermission fPerm = perm;
  if (null == userUgi) {
    files.setPermission(path, perm);
  }
  else {
    userUgi.doAs(new PrivilegedExceptionAction<Void>() {
      public Void run() throws Exception {
        files.setPermission(path, fPerm);
        return null;
      }
    });
  }
  if (isDir) {
    FileStatus[] statuses = fs.listStatus(path);
    for (FileStatus status : statuses) {
      changePermissions(fs, status.getPath());
    }
  }
}
 

@Test(timeout=10000)
@SuppressWarnings("unchecked")
public void testConsistency() {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        null, dispatcher, localrsrc, false, conf,
        new NMNullStateStoreService());

    ResourceEvent req11Event = new ResourceRequestEvent(req1,
        LocalResourceVisibility.PUBLIC, lc1);

    ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();

    // Verify refCount for R1 is 1
    Assert.assertEquals(1, lr1.getRefCount());

    dispatcher.await();
    verifyTrackedResourceCount(tracker, 1);

    // Localize resource1
    ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path(
        "file:///tmp/r1"), 1);
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1")));
    LocalizedResource rsrcbefore = tracker.iterator().next();
    File resFile = new File(lr1.getLocalPath().toUri().getRawPath()
        .toString());
    Assert.assertTrue(resFile.exists());
    Assert.assertTrue(resFile.delete());

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    LocalizedResource rsrcafter = tracker.iterator().next();
    if (rsrcbefore == rsrcafter) {
      Assert.fail("Localized resource should not be equal");
    }
    // Release resource1
    tracker.handle(rel11Event);
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 

@Test(timeout=10000)
@SuppressWarnings("unchecked")
public void test() {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler =
        mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler =
        mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    DeletionService mockDelService = mock(DeletionService.class);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
    LocalizerContext lc2 = new LocalizerContext(user, cId2, null);

    LocalResourceRequest req1 =
        createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
    LocalResourceRequest req2 =
        createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
        new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    localrsrc.put(req2, lr2);
    LocalResourcesTracker tracker =
        new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc,
            false, conf, new NMNullStateStoreService());

    ResourceEvent req11Event =
        new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
    ResourceEvent req12Event =
        new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc2);
    ResourceEvent req21Event =
        new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);

    ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);
    ResourceEvent rel12Event = new ResourceReleaseEvent(req1, cId2);
    ResourceEvent rel21Event = new ResourceReleaseEvent(req2, cId1);

    // Localize R1 for C1
    tracker.handle(req11Event);

    // Localize R1 for C2
    tracker.handle(req12Event);

    // Localize R2 for C1
    tracker.handle(req21Event);

    dispatcher.await();
    verify(localizerEventHandler, times(3)).handle(
        any(LocalizerResourceRequestEvent.class));
    // Verify refCount for R1 is 2
    Assert.assertEquals(2, lr1.getRefCount());
    // Verify refCount for R2 is 1
    Assert.assertEquals(1, lr2.getRefCount());

    // Release R2 for C1
    tracker.handle(rel21Event);

    dispatcher.await();
    verifyTrackedResourceCount(tracker, 2);

    // Verify resource with non zero ref count is not removed.
    Assert.assertEquals(2, lr1.getRefCount());
    Assert.assertFalse(tracker.remove(lr1, mockDelService));
    verifyTrackedResourceCount(tracker, 2);

    // Localize resource1
    ResourceLocalizedEvent rle =
        new ResourceLocalizedEvent(req1, new Path("file:///tmp/r1"), 1);
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));

    // Release resource1
    tracker.handle(rel11Event);
    tracker.handle(rel12Event);
    Assert.assertEquals(0, lr1.getRefCount());

    // Verify resources in state LOCALIZED with ref-count=0 is removed.
    Assert.assertTrue(tracker.remove(lr1, mockDelService));
    verifyTrackedResourceCount(tracker, 1);
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 

@SuppressWarnings("unchecked")
@Test
public void testResourcePresentInGoodDir() throws IOException {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler =
        mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler =
        mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    LocalResourceRequest req1 =
        createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
    LocalResourceRequest req2 =
        createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
        new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    localrsrc.put(req2, lr2);
    LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class);
    List<String> goodDirs = new ArrayList<String>();
    // /tmp/somedir2 is bad
    goodDirs.add("/tmp/somedir1/");
    goodDirs.add("/tmp/somedir2");
    Mockito.when(dirsHandler.getLocalDirs()).thenReturn(goodDirs);
    Mockito.when(dirsHandler.getLocalDirsForRead()).thenReturn(goodDirs);
    LocalResourcesTrackerImpl tracker =
        new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc,
            true , conf, new NMNullStateStoreService(), dirsHandler);
    ResourceEvent req11Event =
        new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
    ResourceEvent req21Event =
        new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
    // Localize R1 for C1
    tracker.handle(req11Event);
    // Localize R2 for C1
    tracker.handle(req21Event);
    dispatcher.await();
    // Localize resource1
    Path p1 = tracker.getPathForLocalization(req1,
        new Path("/tmp/somedir1"), null);
    Path p2 = tracker.getPathForLocalization(req2,
        new Path("/tmp/somedir2"), null);
    ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(req1, p1, 1);
    tracker.handle(rle1);
    ResourceLocalizedEvent rle2 = new ResourceLocalizedEvent(req2, p2, 1);
    tracker.handle(rle2);
    dispatcher.await();
    // Remove somedir2 from gooddirs
    Assert.assertTrue(tracker.checkLocalResource(lr2));
    goodDirs.remove(1);
    Assert.assertFalse(tracker.checkLocalResource(lr2));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 

@Test(timeout=10000)
@SuppressWarnings("unchecked")
public void testConsistency() {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        null, dispatcher, localrsrc, false, conf,
        new NMNullStateStoreService(), null);

    ResourceEvent req11Event = new ResourceRequestEvent(req1,
        LocalResourceVisibility.PUBLIC, lc1);

    ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();

    // Verify refCount for R1 is 1
    Assert.assertEquals(1, lr1.getRefCount());

    dispatcher.await();
    verifyTrackedResourceCount(tracker, 1);

    // Localize resource1
    ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path(
        "file:///tmp/r1"), 1);
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1")));
    LocalizedResource rsrcbefore = tracker.iterator().next();
    File resFile = new File(lr1.getLocalPath().toUri().getRawPath()
        .toString());
    Assert.assertTrue(resFile.exists());
    Assert.assertTrue(resFile.delete());

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    LocalizedResource rsrcafter = tracker.iterator().next();
    if (rsrcbefore == rsrcafter) {
      Assert.fail("Localized resource should not be equal");
    }
    // Release resource1
    tracker.handle(rel11Event);
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 

@Test(timeout=10000)
@SuppressWarnings("unchecked")
public void test() {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler =
        mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler =
        mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    DeletionService mockDelService = mock(DeletionService.class);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
    LocalizerContext lc2 = new LocalizerContext(user, cId2, null);

    LocalResourceRequest req1 =
        createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
    LocalResourceRequest req2 =
        createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
        new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    localrsrc.put(req2, lr2);
    LocalResourcesTracker tracker =
        new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc,
            false, conf, new NMNullStateStoreService(),null);

    ResourceEvent req11Event =
        new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
    ResourceEvent req12Event =
        new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc2);
    ResourceEvent req21Event =
        new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);

    ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);
    ResourceEvent rel12Event = new ResourceReleaseEvent(req1, cId2);
    ResourceEvent rel21Event = new ResourceReleaseEvent(req2, cId1);

    // Localize R1 for C1
    tracker.handle(req11Event);

    // Localize R1 for C2
    tracker.handle(req12Event);

    // Localize R2 for C1
    tracker.handle(req21Event);

    dispatcher.await();
    verify(localizerEventHandler, times(3)).handle(
        any(LocalizerResourceRequestEvent.class));
    // Verify refCount for R1 is 2
    Assert.assertEquals(2, lr1.getRefCount());
    // Verify refCount for R2 is 1
    Assert.assertEquals(1, lr2.getRefCount());

    // Release R2 for C1
    tracker.handle(rel21Event);

    dispatcher.await();
    verifyTrackedResourceCount(tracker, 2);

    // Verify resource with non zero ref count is not removed.
    Assert.assertEquals(2, lr1.getRefCount());
    Assert.assertFalse(tracker.remove(lr1, mockDelService));
    verifyTrackedResourceCount(tracker, 2);

    // Localize resource1
    ResourceLocalizedEvent rle =
        new ResourceLocalizedEvent(req1, new Path("file:///tmp/r1"), 1);
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));

    // Release resource1
    tracker.handle(rel11Event);
    tracker.handle(rel12Event);
    Assert.assertEquals(0, lr1.getRefCount());

    // Verify resources in state LOCALIZED with ref-count=0 is removed.
    Assert.assertTrue(tracker.remove(lr1, mockDelService));
    verifyTrackedResourceCount(tracker, 1);
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 

/**
 * Recursively change permissions of all files/dirs on path based 
 * on resource visibility.
 * Change to 755 or 700 for dirs, 555 or 500 for files.
 * @param fs FileSystem
 * @param path Path to modify perms for
 * @throws IOException
 * @throws InterruptedException 
 */
private void changePermissions(FileSystem fs, final Path path)
    throws IOException, InterruptedException {
  File f = new File(path.toUri());
  if (FileUtils.isSymlink(f)) {
    // avoid following symlinks when changing permissions
    return;
  }
  boolean isDir = f.isDirectory();
  FsPermission perm = cachePerms;
  // set public perms as 755 or 555 based on dir or file
  if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
    perm = isDir ? PUBLIC_DIR_PERMS : PUBLIC_FILE_PERMS;
  }
  // set private perms as 700 or 500
  else {
    // PRIVATE:
    // APPLICATION:
    perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
  }
  LOG.debug("Changing permissions for path " + path
      + " to perm " + perm);
  final FsPermission fPerm = perm;
  if (null == userUgi) {
    files.setPermission(path, perm);
  }
  else {
    userUgi.doAs(new PrivilegedExceptionAction<Void>() {
      public Void run() throws Exception {
        files.setPermission(path, fPerm);
        return null;
      }
    });
  }
  if (isDir) {
    FileStatus[] statuses = fs.listStatus(path);
    for (FileStatus status : statuses) {
      changePermissions(fs, status.getPath());
    }
  }
}