org.apache.hadoop.fs.AbstractFileSystem Java Examples

@Override
public void createSymlink(final Path target, final Path link,
    final boolean createParent) throws IOException, UnresolvedLinkException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(link), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("createSymlink", link);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  res.targetFileSystem.createSymlink(target, res.remainingPath,
      createParent);  
}
 

/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
    primaryFsUri = new URI(primaryFileSystemUriPath());

    primaryFsCfg = new Configuration();

    primaryFsCfg.addResource(U.resolveIgniteUrl(primaryFileSystemConfigPath()));

    UserGroupInformation ugi = UserGroupInformation.getBestUGI(null, getClientFsUser());

    // Create Fs on behalf of the client user:
    ugi.doAs(new PrivilegedExceptionAction<Object>() {
        @Override public Object run() throws Exception {
            fs = AbstractFileSystem.get(primaryFsUri, primaryFsCfg);

            return null;
        }
    });

    barrier = new CyclicBarrier(THREAD_CNT);
}
 

public ChRootedFs(final AbstractFileSystem fs, final Path theRoot)
  throws URISyntaxException {
  super(fs.getUri(), fs.getUri().getScheme(),
      fs.getUri().getAuthority() != null, fs.getUriDefaultPort());
  myFs = fs;
  myFs.checkPath(theRoot);
  chRootPathPart = new Path(myFs.getUriPath(theRoot));
  chRootPathPartString = chRootPathPart.toUri().getPath();
  /*
   * We are making URI include the chrootedPath: e.g. file:///chrootedPath.
   * This is questionable since Path#makeQualified(uri, path) ignores
   * the pathPart of a uri. Since this class is internal we can ignore
   * this issue but if we were to make it external then this needs
   * to be resolved.
   */
  // Handle the two cases:
  //              scheme:/// and scheme://authority/
  myUri = new URI(myFs.getUri().toString() + 
      (myFs.getUri().getAuthority() == null ? "" :  Path.SEPARATOR) +
        chRootPathPart.toUri().getPath().substring(1));
  super.checkPath(theRoot);
}
 

@Override
public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
  List<InodeTree.MountPoint<AbstractFileSystem>> mountPoints = 
              fsState.getMountPoints();
  int initialListSize  = 0;
  for (InodeTree.MountPoint<AbstractFileSystem> im : mountPoints) {
    initialListSize += im.target.targetDirLinkList.length; 
  }
  List<Token<?>> result = new ArrayList<Token<?>>(initialListSize);
  for ( int i = 0; i < mountPoints.size(); ++i ) {
    List<Token<?>> tokens = 
      mountPoints.get(i).target.targetFileSystem.getDelegationTokens(renewer);
    if (tokens != null) {
      result.addAll(tokens);
    }
  }
  return result;
}
 

@Test
public void testGetFileChecksum() throws AccessControlException
  , UnresolvedLinkException, IOException {
  AbstractFileSystem mockAFS = Mockito.mock(AbstractFileSystem.class);
  InodeTree.ResolveResult<AbstractFileSystem> res =
    new InodeTree.ResolveResult<AbstractFileSystem>(null, mockAFS , null,
      new Path("someFile"));
  @SuppressWarnings("unchecked")
  InodeTree<AbstractFileSystem> fsState = Mockito.mock(InodeTree.class);
  Mockito.when(fsState.resolve(Mockito.anyString()
    , Mockito.anyBoolean())).thenReturn(res);
  ViewFs vfs = Mockito.mock(ViewFs.class);
  vfs.fsState = fsState;

  Mockito.when(vfs.getFileChecksum(new Path("/tmp/someFile")))
    .thenCallRealMethod();
  vfs.getFileChecksum(new Path("/tmp/someFile"));

  Mockito.verify(mockAFS).getFileChecksum(new Path("someFile"));
}
 

public ChRootedFs(final AbstractFileSystem fs, final Path theRoot)
  throws URISyntaxException {
  super(fs.getUri(), fs.getUri().getScheme(),
      fs.getUri().getAuthority() != null, fs.getUriDefaultPort());
  myFs = fs;
  myFs.checkPath(theRoot);
  chRootPathPart = new Path(myFs.getUriPath(theRoot));
  chRootPathPartString = chRootPathPart.toUri().getPath();
  /*
   * We are making URI include the chrootedPath: e.g. file:///chrootedPath.
   * This is questionable since Path#makeQualified(uri, path) ignores
   * the pathPart of a uri. Since this class is internal we can ignore
   * this issue but if we were to make it external then this needs
   * to be resolved.
   */
  // Handle the two cases:
  //              scheme:/// and scheme://authority/
  myUri = new URI(myFs.getUri().toString() + 
      (myFs.getUri().getAuthority() == null ? "" :  Path.SEPARATOR) +
        chRootPathPart.toUri().getPath().substring(1));
  super.checkPath(theRoot);
}
 

@Override
public FileStatus[] listStatus(final Path f) throws AccessControlException,
    FileNotFoundException, UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
    fsState.resolve(getUriPath(f), true);
  
  FileStatus[] statusLst = res.targetFileSystem.listStatus(res.remainingPath);
  if (!res.isInternalDir()) {
    // We need to change the name in the FileStatus as described in
    // {@link #getFileStatus }
    ChRootedFs targetFs;
    targetFs = (ChRootedFs) res.targetFileSystem;
    int i = 0;
    for (FileStatus status : statusLst) {
        String suffix = targetFs.stripOutRoot(status.getPath());
        statusLst[i++] = new ViewFsFileStatus(status, this.makeQualified(
            suffix.length() == 0 ? f : new Path(res.resolvedPath, suffix)));
    }
  }
  return statusLst;
}
 

@Override
public FileStatus[] listStatus(final Path f) throws AccessControlException,
    FileNotFoundException, UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
    fsState.resolve(getUriPath(f), true);
  
  FileStatus[] statusLst = res.targetFileSystem.listStatus(res.remainingPath);
  if (!res.isInternalDir()) {
    // We need to change the name in the FileStatus as described in
    // {@link #getFileStatus }
    ChRootedFs targetFs;
    targetFs = (ChRootedFs) res.targetFileSystem;
    int i = 0;
    for (FileStatus status : statusLst) {
        String suffix = targetFs.stripOutRoot(status.getPath());
        statusLst[i++] = new ViewFsFileStatus(status, this.makeQualified(
            suffix.length() == 0 ? f : new Path(res.resolvedPath, suffix)));
    }
  }
  return statusLst;
}
 

@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 

@Override
public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
  List<InodeTree.MountPoint<AbstractFileSystem>> mountPoints = 
              fsState.getMountPoints();
  int initialListSize  = 0;
  for (InodeTree.MountPoint<AbstractFileSystem> im : mountPoints) {
    initialListSize += im.target.targetDirLinkList.length; 
  }
  List<Token<?>> result = new ArrayList<Token<?>>(initialListSize);
  for ( int i = 0; i < mountPoints.size(); ++i ) {
    List<Token<?>> tokens = 
      mountPoints.get(i).target.targetFileSystem.getDelegationTokens(renewer);
    if (tokens != null) {
      result.addAll(tokens);
    }
  }
  return result;
}
 

@Test
public void testGetFileChecksum() throws AccessControlException
  , UnresolvedLinkException, IOException {
  AbstractFileSystem mockAFS = Mockito.mock(AbstractFileSystem.class);
  InodeTree.ResolveResult<AbstractFileSystem> res =
    new InodeTree.ResolveResult<AbstractFileSystem>(null, mockAFS , null,
      new Path("someFile"));
  @SuppressWarnings("unchecked")
  InodeTree<AbstractFileSystem> fsState = Mockito.mock(InodeTree.class);
  Mockito.when(fsState.resolve(Mockito.anyString()
    , Mockito.anyBoolean())).thenReturn(res);
  ViewFs vfs = Mockito.mock(ViewFs.class);
  vfs.fsState = fsState;

  Mockito.when(vfs.getFileChecksum(new Path("/tmp/someFile")))
    .thenCallRealMethod();
  vfs.getFileChecksum(new Path("/tmp/someFile"));

  Mockito.verify(mockAFS).getFileChecksum(new Path("someFile"));
}
 

@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}
 

@Override
public FileStatus getFileStatus(final Path f) throws AccessControlException,
    FileNotFoundException, UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);

  //  FileStatus#getPath is a fully qualified path relative to the root of 
  // target file system.
  // We need to change it to viewfs URI - relative to root of mount table.
  
  // The implementors of RawLocalFileSystem were trying to be very smart.
  // They implement FileStatus#getOwener lazily -- the object
  // returned is really a RawLocalFileSystem that expect the
  // FileStatus#getPath to be unchanged so that it can get owner when needed.
  // Hence we need to interpose a new ViewFsFileStatus that works around.
  
  
  FileStatus status =  res.targetFileSystem.getFileStatus(res.remainingPath);
  return new ViewFsFileStatus(status, this.makeQualified(f));
}
 

@Override
public void createSymlink(final Path target, final Path link,
    final boolean createParent) throws IOException, UnresolvedLinkException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(link), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("createSymlink", link);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  res.targetFileSystem.createSymlink(target, res.remainingPath,
      createParent);  
}
 

@Override
public void setPermission(final Path f, final FsPermission permission)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);
  res.targetFileSystem.setPermission(res.remainingPath, permission); 
  
}
 

public MountPoint[] getMountPoints() {
  List<InodeTree.MountPoint<AbstractFileSystem>> mountPoints = 
                fsState.getMountPoints();
  
  MountPoint[] result = new MountPoint[mountPoints.size()];
  for ( int i = 0; i < mountPoints.size(); ++i ) {
    result[i] = new MountPoint(new Path(mountPoints.get(i).src), 
                            mountPoints.get(i).target.targetDirLinkList);
  }
  return result;
}
 

@Test
public void testIsValidNameValidInBaseFs() throws Exception {
  AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
  ChRootedFs chRootedFs = new ChRootedFs(baseFs, new Path("/chroot"));
  Mockito.doReturn(true).when(baseFs).isValidName(Mockito.anyString());
  Assert.assertTrue(chRootedFs.isValidName("/test"));
  Mockito.verify(baseFs).isValidName("/chroot/test");
}
 

@Override
public boolean setReplication(final Path f, final short replication)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);
  return res.targetFileSystem.setReplication(res.remainingPath, replication);
}
 

@Override
public void setTimes(final Path f, final long mtime, final long atime)
    throws AccessControlException, FileNotFoundException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res = 
    fsState.resolve(getUriPath(f), true);
  res.targetFileSystem.setTimes(res.remainingPath, mtime, atime); 
}
 

/**
 * Check path does not exist in a given FileSystem.
 *
 * @param fs FileSystem to check.
 * @param path Path to check.
 */
private void assertPathDoesNotExist(final AbstractFileSystem fs, final Path path) {
    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            return fs.getFileStatus(path);
        }
    }, FileNotFoundException.class, null);
}
 

@Override
public void removeDefaultAcl(Path path)
    throws IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
      fsState.resolve(getUriPath(path), true);
  res.targetFileSystem.removeDefaultAcl(res.remainingPath);
}
 

@Override
public Path resolvePath(final Path f) throws FileNotFoundException,
        AccessControlException, UnresolvedLinkException, IOException {
  final InodeTree.ResolveResult<AbstractFileSystem> res;
    res = fsState.resolve(getUriPath(f), true);
  if (res.isInternalDir()) {
    return f;
  }
  return res.targetFileSystem.resolvePath(res.remainingPath);

}
 

/**
 * This constructor has the signature needed by
 * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}.
 * 
 * @param theUri which must be that of ViewFs
 * @param conf
 * @throws IOException
 * @throws URISyntaxException 
 */
ViewFs(final URI theUri, final Configuration conf) throws IOException,
    URISyntaxException {
  super(theUri, FsConstants.VIEWFS_SCHEME, false, -1);
  creationTime = Time.now();
  ugi = UserGroupInformation.getCurrentUser();
  config = conf;
  // Now build  client side view (i.e. client side mount table) from config.
  String authority = theUri.getAuthority();
  fsState = new InodeTree<AbstractFileSystem>(conf, authority) {

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(final URI uri)
      throws URISyntaxException, UnsupportedFileSystemException {
        String pathString = uri.getPath();
        if (pathString.isEmpty()) {
          pathString = "/";
        }
        return new ChRootedFs(
            AbstractFileSystem.createFileSystem(uri, config),
            new Path(pathString));
    }

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(
        final INodeDir<AbstractFileSystem> dir) throws URISyntaxException {
      return new InternalDirOfViewFs(dir, creationTime, ugi, getUri());
    }

    @Override
    protected
    AbstractFileSystem getTargetFileSystem(URI[] mergeFsURIList)
        throws URISyntaxException, UnsupportedFileSystemException {
      throw new UnsupportedFileSystemException("mergefs not implemented yet");
      // return MergeFs.createMergeFs(mergeFsURIList, config);
    }
  };
}
 

@Override
public RemoteIterator<FileStatus> listStatusIterator(final Path f)
  throws AccessControlException, FileNotFoundException,
  UnresolvedLinkException, IOException {
  final InodeTree.ResolveResult<AbstractFileSystem> res =
    fsState.resolve(getUriPath(f), true);
  final RemoteIterator<FileStatus> fsIter =
    res.targetFileSystem.listStatusIterator(res.remainingPath);
  if (res.isInternalDir()) {
    return fsIter;
  }
  
  return new RemoteIterator<FileStatus>() {
    final RemoteIterator<FileStatus> myIter;
    final ChRootedFs targetFs;
    { // Init
        myIter = fsIter;
        targetFs = (ChRootedFs) res.targetFileSystem;
    }
    
    @Override
    public boolean hasNext() throws IOException {
      return myIter.hasNext();
    }
    
    @Override
    public FileStatus next() throws IOException {
      FileStatus status =  myIter.next();
      String suffix = targetFs.stripOutRoot(status.getPath());
      return new ViewFsFileStatus(status, makeQualified(
          suffix.length() == 0 ? f : new Path(res.resolvedPath, suffix)));
    }
  };
}
 

@Test
public void testFailedDirLogDeletion() throws Exception {

  File[] localLogDirs = getLocalLogDirFiles(this.getClass().getName(), 7);
  final List<String> localLogDirPaths =
      new ArrayList<String>(localLogDirs.length);
  for (int i = 0; i < localLogDirs.length; i++) {
    localLogDirPaths.add(localLogDirs[i].getAbsolutePath());
  }

  String localLogDirsString = StringUtils.join(localLogDirPaths, ",");

  conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString);
  conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 0l);

  LocalDirsHandlerService mockDirsHandler = mock(LocalDirsHandlerService.class);

  NonAggregatingLogHandler rawLogHandler =
      new NonAggregatingLogHandler(dispatcher, mockDelService,
          mockDirsHandler, new NMNullStateStoreService());
  NonAggregatingLogHandler logHandler = spy(rawLogHandler);
  AbstractFileSystem spylfs =
      spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
  FileContext lfs = FileContext.getFileContext(spylfs, conf);
  doReturn(lfs).when(logHandler)
    .getLocalFileContext(isA(Configuration.class));
  logHandler.init(conf);
  logHandler.start();
  runMockedFailedDirs(logHandler, appId, user, mockDelService,
    mockDirsHandler, conf, spylfs, lfs, localLogDirs);
  logHandler.close();
}
 

/**
 * Deletes all files from the given file system.
 *
 * @param fs The file system to clean up.
 * @throws IOException On error.
 */
public static void clear(AbstractFileSystem fs) throws IOException {
    // Delete root contents:
    FileStatus[] statuses = fs.listStatus(new Path("/"));

    if (statuses != null) {
        for (FileStatus stat: statuses)
            fs.delete(stat.getPath(), true);
    }
}
 

@Test
public void testIsValidNameInvalidInBaseFs() throws Exception {
  AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
  ChRootedFs chRootedFs = new ChRootedFs(baseFs, new Path("/chroot"));
  Mockito.doReturn(false).when(baseFs).isValidName(Mockito.anyString());
  Assert.assertFalse(chRootedFs.isValidName("/test"));
  Mockito.verify(baseFs).isValidName("/chroot/test");
}
 

@Test
public void testIsValidNameValidInBaseFs() throws Exception {
  AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
  ChRootedFs chRootedFs = new ChRootedFs(baseFs, new Path("/chroot"));
  Mockito.doReturn(true).when(baseFs).isValidName(Mockito.anyString());
  Assert.assertTrue(chRootedFs.isValidName("/test"));
  Mockito.verify(baseFs).isValidName("/chroot/test");
}
 

@Override
public FileStatus[] listStatus(final Path f) throws AccessControlException,
    IOException {
  checkPathIsSlash(f);
  FileStatus[] result = new FileStatus[theInternalDir.children.size()];
  int i = 0;
  for (Entry<String, INode<AbstractFileSystem>> iEntry : 
                                      theInternalDir.children.entrySet()) {
    INode<AbstractFileSystem> inode = iEntry.getValue();

    
    if (inode instanceof INodeLink ) {
      INodeLink<AbstractFileSystem> link = 
        (INodeLink<AbstractFileSystem>) inode;

      result[i++] = new FileStatus(0, false, 0, 0,
        creationTime, creationTime,
        PERMISSION_555, ugi.getUserName(), ugi.getGroupNames()[0],
        link.getTargetLink(),
        new Path(inode.fullPath).makeQualified(
            myUri, null));
    } else {
      result[i++] = new FileStatus(0, true, 0, 0,
        creationTime, creationTime,
        PERMISSION_555, ugi.getUserName(), ugi.getGroupNames()[0],
        new Path(inode.fullPath).makeQualified(
            myUri, null));
    }
  }
  return result;
}
 

@Override
public Map<String, byte[]> getXAttrs(Path path, List<String> names)
    throws IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res =
      fsState.resolve(getUriPath(path), true);
  return res.targetFileSystem.getXAttrs(res.remainingPath, names);
}