Java Code Examples for org.apache.hadoop.fs.FileStatus#isSymlink()

/**
 * Validates the toString method for FileStatus.
 * @param fileStatus FileStatus to be validated
 */
private void validateToString(FileStatus fileStatus) throws IOException {
  StringBuilder expected = new StringBuilder();
  expected.append("FileStatus{");
  expected.append("path=").append(fileStatus.getPath()).append("; ");
  expected.append("isDirectory=").append(fileStatus.isDirectory()).append("; ");
  if(!fileStatus.isDirectory()) {
    expected.append("length=").append(fileStatus.getLen()).append("; ");
    expected.append("replication=").append(fileStatus.getReplication()).append("; ");
    expected.append("blocksize=").append(fileStatus.getBlockSize()).append("; ");
  }
  expected.append("modification_time=").append(fileStatus.getModificationTime()).append("; ");
  expected.append("access_time=").append(fileStatus.getAccessTime()).append("; ");
  expected.append("owner=").append(fileStatus.getOwner()).append("; ");
  expected.append("group=").append(fileStatus.getGroup()).append("; ");
  expected.append("permission=").append(fileStatus.getPermission()).append("; ");
  if(fileStatus.isSymlink()) {
    expected.append("isSymlink=").append(true).append("; ");
    expected.append("symlink=").append(fileStatus.getSymlink()).append("}");
  } else {
    expected.append("isSymlink=").append(false).append("}");
  }
  
  assertEquals(expected.toString(), fileStatus.toString());
}
 

public static FileStatusDTO fromFileStatus(FileStatus fileStatus) {
  if (null == fileStatus) {
    return null;
  }

  FileStatusDTO dto = new FileStatusDTO();
  try {
    dto.path = FilePathDTO.fromPath(fileStatus.getPath());
    dto.length = fileStatus.getLen();
    dto.isdir = fileStatus.isDirectory();
    dto.blockReplication = fileStatus.getReplication();
    dto.blocksize = fileStatus.getBlockSize();
    dto.modificationTime = fileStatus.getModificationTime();
    dto.accessTime = fileStatus.getModificationTime();
    dto.symlink = fileStatus.isSymlink() ? FilePathDTO.fromPath(fileStatus.getSymlink()) : null;
    safeReadAndSetMetadata(dto, fileStatus);
  } catch (IOException ioe) {
    throw new HoodieException(ioe);
  }
  return dto;
}
 

/**
 * Validates the toString method for FileStatus.
 * @param fileStatus FileStatus to be validated
 */
private void validateToString(FileStatus fileStatus) throws IOException {
  StringBuilder expected = new StringBuilder();
  expected.append("FileStatus{");
  expected.append("path=").append(fileStatus.getPath()).append("; ");
  expected.append("isDirectory=").append(fileStatus.isDirectory()).append("; ");
  if(!fileStatus.isDirectory()) {
    expected.append("length=").append(fileStatus.getLen()).append("; ");
    expected.append("replication=").append(fileStatus.getReplication()).append("; ");
    expected.append("blocksize=").append(fileStatus.getBlockSize()).append("; ");
  }
  expected.append("modification_time=").append(fileStatus.getModificationTime()).append("; ");
  expected.append("access_time=").append(fileStatus.getAccessTime()).append("; ");
  expected.append("owner=").append(fileStatus.getOwner()).append("; ");
  expected.append("group=").append(fileStatus.getGroup()).append("; ");
  expected.append("permission=").append(fileStatus.getPermission()).append("; ");
  if(fileStatus.isSymlink()) {
    expected.append("isSymlink=").append(true).append("; ");
    expected.append("symlink=").append(fileStatus.getSymlink()).append("}");
  } else {
    expected.append("isSymlink=").append(false).append("}");
  }
  
  assertEquals(expected.toString(), fileStatus.toString());
}
 

private void verifyPermsRecursively(FileSystem fs,
    FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  FileStatus status = files.getFileStatus(p);
  if (status.isDirectory()) {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_DIR_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_DIR_PERMS.toShort());
    }
    if (!status.isSymlink()) {
      FileStatus[] statuses = fs.listStatus(p);
      for (FileStatus stat : statuses) {
        verifyPermsRecursively(fs, files, stat.getPath(), vis);
      }
    }
  }
  else {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_FILE_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_FILE_PERMS.toShort());
    }
  }      
}
 

/**
 * Returns the {@link FileStatus} from the {@link PathData} item.
 * If the current options require links to be followed then the
 * returned file status is that of the linked file.
 * @param item PathData
 * @return FileStatus
 */
protected FileStatus getFileStatus(PathData item) throws IOException {
  FileStatus fileStatus = item.stat;
  if(getOptions().isFollowLink() && fileStatus.isSymlink()) {
    Path linkedFile = fileStatus.getSymlink();
    fileStatus = getFileSystem(item).getFileStatus(linkedFile);
  }
  return fileStatus;
}
 

/**
 * Replace the scheme of the input {@link FileStatus} if it matches the string to replace.
 */
public static FileStatus replaceScheme(FileStatus st, String replace, String replacement) {
  if (replace != null && replace.equals(replacement)) {
    return st;
  }
  try {
    return new FileStatus(st.getLen(), st.isDir(), st.getReplication(), st.getBlockSize(), st.getModificationTime(),
        st.getAccessTime(), st.getPermission(), st.getOwner(), st.getGroup(), st.isSymlink() ? st.getSymlink() : null,
        replaceScheme(st.getPath(), replace, replacement));
  } catch (IOException ioe) {
    throw new RuntimeException(ioe);
  }
}
 

public FileStatusEntity(FileStatus status) throws IOException {
    //this.path = status.getPath();
    this.length = status.getLen();
    this.isdir = status.isDirectory();
    this.block_replication = status.getReplication();
    this.blocksize = status.getBlockSize();
    this.modification_time = status.getModificationTime();
    this.access_time = status.getAccessTime();
    this.permission = status.getPermission();
    this.owner = status.getOwner();
    this.group = status.getGroup();
    if(status.isSymlink()) {
        this.symlink = status.getSymlink();
    }
}
 

public FileStatusEntity(FileStatus status) throws IOException {
    //this.path = status.getPath();
    this.length = status.getLen();
    this.isdir = status.isDirectory();
    this.block_replication = status.getReplication();
    this.blocksize = status.getBlockSize();
    this.modification_time = status.getModificationTime();
    this.access_time = status.getAccessTime();
    this.permission = status.getPermission();
    this.owner = status.getOwner();
    this.group = status.getGroup();
    if(status.isSymlink()) {
        this.symlink = status.getSymlink();
    }
}
 

/**
 * Returns the {@link FileStatus} from the {@link PathData} item. If the
 * current options require links to be followed then the returned file status
 * is that of the linked file.
 *
 * @param item
 *          PathData
 * @param depth
 *          current depth in the process directories
 * @return FileStatus
 */
protected FileStatus getFileStatus(PathData item, int depth)
    throws IOException {
  FileStatus fileStatus = item.stat;
  if (fileStatus.isSymlink()) {
    if (options.isFollowLink() || (options.isFollowArgLink() &&
        (depth == 0))) {
      Path linkedFile = item.fs.resolvePath(fileStatus.getSymlink());
      fileStatus = getFileSystem(item).getFileStatus(linkedFile);
    }
  }
  return fileStatus;
}
 

public static FILE_TYPE getType(FileStatus fileStatus) {
  if (fileStatus.isFile()) {
    return FILE;
  }
  if (fileStatus.isDirectory()) {
    return DIRECTORY;
  }
  if (fileStatus.isSymlink()) {
    return SYMLINK;
  }
  throw new IllegalArgumentException("Could not determine filetype for: " +
                                     fileStatus.getPath());
}
 

private void verifyPermsRecursively(FileSystem fs,
    FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  FileStatus status = files.getFileStatus(p);
  if (status.isDirectory()) {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_DIR_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_DIR_PERMS.toShort());
    }
    if (!status.isSymlink()) {
      FileStatus[] statuses = fs.listStatus(p);
      for (FileStatus stat : statuses) {
        verifyPermsRecursively(fs, files, stat.getPath(), vis);
      }
    }
  }
  else {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_FILE_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_FILE_PERMS.toShort());
    }
  }      
}
 

/**
 * Converts a Hadoop {@link FileStatus} instance into a protobuf
 * {@link DFSProtos.FileStatus}
 *
 * @param status
 *          the Hadoop status instance to convert
 * @return a protobuf status instance
 * @throws IOException
 */
static DFS.FileStatus toProtoFileStatus(FileStatus status) throws IOException {
  DFS.FileStatus.Builder builder = DFS.FileStatus.newBuilder();

  builder
    .setLength(status.getLen())
    .setIsDirectory(status.isDirectory())
    .setBlockReplication(status.getReplication())
    .setBlockSize(status.getBlockSize())
    .setModificationTime(status.getModificationTime())
    .setAccessTime(status.getAccessTime());

  // Handling potential null values
  if (status.getPath() != null) {
    builder = builder.setPath(status.getPath().toUri().getPath());
  }
  if (status.getPermission() != null) {
    builder = builder.setPermission(status.getPermission().toExtendedShort());
  }
  if (status.getOwner() != null) {
    builder = builder.setOwner(status.getOwner());
  }
  if (status.getGroup() != null) {
    builder = builder.setGroup(status.getGroup());
  }
  if (status.isSymlink()) {
    builder = builder.setSymlink(status.getSymlink().toString());
  }

  return builder.build();
}
 

/**
 * Returns the {@link FileStatus} from the {@link PathData} item. If the
 * current options require links to be followed then the returned file status
 * is that of the linked file.
 *
 * @param item
 *          PathData
 * @param depth
 *          current depth in the process directories
 * @return FileStatus
 */
protected FileStatus getFileStatus(PathData item, int depth)
    throws IOException {
  FileStatus fileStatus = item.stat;
  if (fileStatus.isSymlink()) {
    if (options.isFollowLink() || (options.isFollowArgLink() &&
        (depth == 0))) {
      Path linkedFile = item.fs.resolvePath(fileStatus.getSymlink());
      fileStatus = getFileSystem(item).getFileStatus(linkedFile);
    }
  }
  return fileStatus;
}
 

public static FILE_TYPE getType(FileStatus fileStatus) {
  if (fileStatus.isFile()) {
    return FILE;
  }
  if (fileStatus.isDirectory()) {
    return DIRECTORY;
  }
  if (fileStatus.isSymlink()) {
    return SYMLINK;
  }
  throw new IllegalArgumentException("Could not determine filetype for: " +
                                     fileStatus.getPath());
}
 

public boolean matches(DrillFileSystem fs, FileStatus status) throws IOException{
  if (ranges.isEmpty() || status.isDirectory()) {
    return false;
  }
  // walk all the way down in the symlinks until a hard entry is reached
  FileStatus current = status;
  while (current.isSymlink()) {
    current = fs.getFileStatus(status.getSymlink());
  }
  // if hard entry is not a file nor can it be a symlink then it is not readable simply deny matching.
  if (!current.isFile()) {
    return false;
  }

  final Range<Long> fileRange = Range.closedOpen( 0L, status.getLen());

  try (FSDataInputStream is = fs.open(status.getPath())) {
    for(RangeMagics rMagic : ranges) {
      Range<Long> r = rMagic.range;
      if (!fileRange.encloses(r)) {
        continue;
      }
      int len = (int) (r.upperEndpoint() - r.lowerEndpoint());
      byte[] bytes = new byte[len];
      is.readFully(r.lowerEndpoint(), bytes);
      for (byte[] magic : rMagic.magics) {
        if (Arrays.equals(magic, bytes)) {
          return true;
        }
      }
    }
  }
  return false;
}
 

/**
 * Move blocks from srcs to trg and delete srcs afterwards.
 * The file block sizes must be the same.
 * 
 * @param trg existing file to append to
 * @param psrcs list of files (same block size, same replication)
 * @throws IOException
 */
@Override
public void concat(Path trg, Path [] psrcs) throws IOException {
  statistics.incrementWriteOps(1);
  // Make target absolute
  Path absF = fixRelativePart(trg);
  // Make all srcs absolute
  Path[] srcs = new Path[psrcs.length];
  for (int i=0; i<psrcs.length; i++) {
    srcs[i] = fixRelativePart(psrcs[i]);
  }
  // Try the concat without resolving any links
  String[] srcsStr = new String[psrcs.length];
  try {
    for (int i=0; i<psrcs.length; i++) {
      srcsStr[i] = getPathName(srcs[i]);
    }
    dfs.concat(getPathName(trg), srcsStr);
  } catch (UnresolvedLinkException e) {
    // Exception could be from trg or any src.
    // Fully resolve trg and srcs. Fail if any of them are a symlink.
    FileStatus stat = getFileLinkStatus(absF);
    if (stat.isSymlink()) {
      throw new IOException("Cannot concat with a symlink target: "
          + trg + " -> " + stat.getPath());
    }
    absF = fixRelativePart(stat.getPath());
    for (int i=0; i<psrcs.length; i++) {
      stat = getFileLinkStatus(srcs[i]);
      if (stat.isSymlink()) {
        throw new IOException("Cannot concat with a symlink src: "
            + psrcs[i] + " -> " + stat.getPath());
      }
      srcs[i] = fixRelativePart(stat.getPath());
    }
    // Try concat again. Can still race with another symlink.
    for (int i=0; i<psrcs.length; i++) {
      srcsStr[i] = getPathName(srcs[i]);
    }
    dfs.concat(getPathName(absF), srcsStr);
  }
}
 

/**
 * Move blocks from srcs to trg and delete srcs afterwards.
 * The file block sizes must be the same.
 * 
 * @param trg existing file to append to
 * @param psrcs list of files (same block size, same replication)
 * @throws IOException
 */
@Override
public void concat(Path trg, Path [] psrcs) throws IOException {
  statistics.incrementWriteOps(1);
  // Make target absolute
  Path absF = fixRelativePart(trg);
  // Make all srcs absolute
  Path[] srcs = new Path[psrcs.length];
  for (int i=0; i<psrcs.length; i++) {
    srcs[i] = fixRelativePart(psrcs[i]);
  }
  // Try the concat without resolving any links
  String[] srcsStr = new String[psrcs.length];
  try {
    for (int i=0; i<psrcs.length; i++) {
      srcsStr[i] = getPathName(srcs[i]);
    }
    dfs.concat(getPathName(trg), srcsStr);
  } catch (UnresolvedLinkException e) {
    // Exception could be from trg or any src.
    // Fully resolve trg and srcs. Fail if any of them are a symlink.
    FileStatus stat = getFileLinkStatus(absF);
    if (stat.isSymlink()) {
      throw new IOException("Cannot concat with a symlink target: "
          + trg + " -> " + stat.getPath());
    }
    absF = fixRelativePart(stat.getPath());
    for (int i=0; i<psrcs.length; i++) {
      stat = getFileLinkStatus(srcs[i]);
      if (stat.isSymlink()) {
        throw new IOException("Cannot concat with a symlink src: "
            + psrcs[i] + " -> " + stat.getPath());
      }
      srcs[i] = fixRelativePart(stat.getPath());
    }
    // Try concat again. Can still race with another symlink.
    for (int i=0; i<psrcs.length; i++) {
      srcsStr[i] = getPathName(srcs[i]);
    }
    dfs.concat(getPathName(absF), srcsStr);
  }
}
 

public boolean matches(FileStatus stat) {
  return stat.isSymlink();
}