Java Code Examples for org.apache.hadoop.fs.FileSystem#getFileStatus()

@Override
protected LocatedFileStatus[] listLocatedStatus(JobConf job) throws IOException {
  Path[] files = getInputPaths(job);
  LocatedFileStatus[] results = new LocatedFileStatus[files.length];
  for (int i = 0; i < files.length; i++) {
    Path p = files[i];
    FileSystem fs = p.getFileSystem(job);
    FileStatus stat = fs.getFileStatus(p);
    if (stat.isDir()) {
      results[i] = new LocatedFileStatus(stat, null);
    } else {
      results[i] = new LocatedFileStatus(stat,
          fs.getFileBlockLocations(stat, 0, stat.getLen()));
    }
  }
  return results;
}
 

private void testCreate(Path path, boolean override) throws Exception {
  FileSystem fs = getHttpFSFileSystem();
  FsPermission permission = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE);
  OutputStream os = fs.create(new Path(path.toUri().getPath()), permission, override, 1024,
                              (short) 2, 100 * 1024 * 1024, null);
  os.write(1);
  os.close();
  fs.close();

  fs = FileSystem.get(getProxiedFSConf());
  FileStatus status = fs.getFileStatus(path);
  if (!isLocalFS()) {
    Assert.assertEquals(status.getReplication(), 2);
    Assert.assertEquals(status.getBlockSize(), 100 * 1024 * 1024);
  }
  Assert.assertEquals(status.getPermission(), permission);
  InputStream is = fs.open(path);
  Assert.assertEquals(is.read(), 1);
  is.close();
  fs.close();
}
 

public static boolean isFileEmpty(FileSystem fs, Path dir) throws IOException {
	FileStatus fstat = fs.getFileStatus(dir);

	if( fstat.isDirectory() 
		|| IOUtilFunctions.isObjectStoreFileScheme(dir) )
	{
		// it is a directory
		FileStatus[] stats = fs.listStatus(dir);
		if (stats != null) {
			for (FileStatus stat : stats) {
				if (stat.getLen() > 0)
					return false;
			}
			return true;
		} else {
			return true;
		}
	} 
	else {
		// it is a regular file
		return (fstat.getLen() == 0);
	}
}
 

private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}
 

private void produceCloseFileEvent(FileSystem fs, Path finalPath) throws IOException {
  FileStatus status = fs.getFileStatus(finalPath);
  HdfsEvents.CLOSED_FILE.create(context)
    .with("filepath", finalPath.toString())
    .with("filename", finalPath.getName())
    .with("length", status.getLen())
    .createAndSend();

  LineageEvent event = context.createLineageEvent(LineageEventType.ENTITY_CREATED);
  event.setSpecificAttribute(LineageSpecificAttribute.ENDPOINT_TYPE, EndPointType.HDFS.name());
  event.setSpecificAttribute(LineageSpecificAttribute.ENTITY_NAME, finalPath.toString());
  event.setSpecificAttribute(LineageSpecificAttribute.DESCRIPTION, finalPath.toString());
  context.publishLineageEvent(event);
}
 

public static void checkFullFile(FileSystem fs, Path name, int len,
    final byte[] compareContent, String message,
    boolean checkFileStatus) throws IOException {
  if (checkFileStatus) {
    final FileStatus status = fs.getFileStatus(name);
    assertEquals("len=" + len + " but status.getLen()=" + status.getLen(),
        len, status.getLen());
  }

  FSDataInputStream stm = fs.open(name);
  byte[] actual = new byte[len];
  stm.readFully(0, actual);
  checkData(actual, 0, compareContent, message);
  stm.close();
}
 

public static LocalResource createApplicationResource(FileSystem fs, Path path, LocalResourceType type)
    throws IOException {
  LocalResource localResource = Records.newRecord(LocalResource.class);
  FileStatus fileStatus = fs.getFileStatus(path);
  localResource.setResource(ConverterUtils.getYarnUrlFromPath(path));
  localResource.setSize(fileStatus.getLen());
  localResource.setTimestamp(fileStatus.getModificationTime());
  localResource.setType(type);
  localResource.setVisibility(LocalResourceVisibility.APPLICATION);
  return localResource;
}
 

@Test
public void testPreservePermissionOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.PERMISSION);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
 

void checkGetBlockLocationsWorks(FileSystem fs, Path fileName) throws IOException {
  FileStatus stat = fs.getFileStatus(fileName);
  try {  
    fs.getFileBlockLocations(stat, 0, 1000);
  } catch (SafeModeException e) {
    assertTrue("Should have not got safemode exception", false);
  } catch (RemoteException re) {
    assertTrue("Should have not got safemode exception", false);   
  }    
}
 

@Test
public void testPreservePermissionOnDirectory() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.PERMISSION);

  Path dst = new Path("/tmp/abc");
  Path src = new Path("/tmp/src");

  createDirectory(fs, src);
  createDirectory(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
}
 

/**
 * Assert that a path does not exist
 *
 * @param fileSystem filesystem to examine
 * @param message message to include in the assertion failure message
 * @param path path in the filesystem
 * @throws IOException IO problems
 */
public static void assertPathDoesNotExist(FileSystem fileSystem,
                                          String message,
                                          Path path) throws IOException {
  try {
    FileStatus status = fileSystem.getFileStatus(path);
    fail(message + ": unexpectedly found " + path + " as  " + status);
  } catch (FileNotFoundException expected) {
    //this is expected

  }
}
 

/**
 * Verify or create the Staging area directory on the configured Filesystem
 * @param stagingArea Staging area directory path
 * @return the FileSytem for the staging area directory
 * @throws IOException
 */
public static FileSystem ensureStagingDirExists(Configuration conf,
    Path stagingArea)
    throws IOException {
  FileSystem fs = stagingArea.getFileSystem(conf);
  String realUser;
  String currentUser;
  UserGroupInformation ugi = UserGroupInformation.getLoginUser();
  realUser = ugi.getShortUserName();
  currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
  if (fs.exists(stagingArea)) {
    FileStatus fsStatus = fs.getFileStatus(stagingArea);
    String owner = fsStatus.getOwner();
    if (!(owner.equals(currentUser) || owner.equals(realUser))) {
      throw new IOException("The ownership on the staging directory "
          + stagingArea + " is not as expected. " + "It is owned by " + owner
          + ". The directory must " + "be owned by the submitter "
          + currentUser + " or " + "by " + realUser);
    }
    if (!fsStatus.getPermission().equals(TezCommonUtils.TEZ_AM_DIR_PERMISSION)) {
      LOG.info("Permissions on staging directory " + stagingArea + " are "
          + "incorrect: " + fsStatus.getPermission()
          + ". Fixing permissions " + "to correct value "
          + TezCommonUtils.TEZ_AM_DIR_PERMISSION);
      fs.setPermission(stagingArea, TezCommonUtils.TEZ_AM_DIR_PERMISSION);
    }
  } else {
    TezCommonUtils.mkDirForAM(fs, stagingArea);
  }
  return fs;
}
 

public static Path readRealPathDataFromSymlinkPath(FileSystem fileSystem, Path linkPath) throws IOException,
    UnsupportedEncodingException {
  FileStatus fileStatus = fileSystem.getFileStatus(linkPath);
  FSDataInputStream inputStream = fileSystem.open(linkPath);
  byte[] buf = new byte[(int) fileStatus.getLen()];
  inputStream.readFully(buf);
  inputStream.close();
  Path path = new Path(new String(buf, UTF_8));
  return path;
}
 

private static void addFileToClasspath(Configuration conf, Set<String> existing, FileSystem fs, Path hdfsBase,
    File file) throws IOException
{
  Path hdfsPath = new Path(hdfsBase, file.getName());
  if (!existing.contains(hdfsPath.toString()))
  {
    if (fs.exists(hdfsPath))
    {
      // check the timestamp and exit if the one in hdfs is "newer"
      FileStatus status = fs.getFileStatus(hdfsPath);

      if (file.lastModified() <= status.getModificationTime())
      {
        log.debug(file.getPath() + " up to date");
        DistributedCache.addFileToClassPath(hdfsPath, conf, fs);

        existing.add(hdfsPath.toString());
        return;
      }
    }

    // copy the file...
    log.debug("Copying " + file.getPath() + " to HDFS for distribution");

    fs.copyFromLocalFile(new Path(file.getCanonicalFile().toURI()), hdfsPath);
    DistributedCache.addFileToClassPath(hdfsPath, conf, fs);
    existing.add(hdfsPath.toString());
  }
}
 

@Override
protected Path createInputFileListing(Job job) throws IOException {

  if (conf.get(NUMBER_OF_LEVELS_TO_PRESERVE_KEY) == null) {
    return super.createInputFileListing(job);
  }
  long totalBytesExpected = 0;
  int totalRecords = 0;
  Path fileListingPath = getFileListingPath();
  try (SequenceFile.Writer writer = getWriter(fileListingPath)) {
    List<Path> srcFiles = getSourceFiles();
    if (srcFiles.size() == 0) {
      return fileListingPath;
    }
    totalRecords = srcFiles.size();
    FileSystem fs = srcFiles.get(0).getFileSystem(conf);
    for (Path path : srcFiles) {
      FileStatus fst = fs.getFileStatus(path);
      totalBytesExpected += fst.getLen();
      Text key = getKey(path);
      writer.append(key, new CopyListingFileStatus(fst));
    }
    writer.close();

    // update jobs configuration

    Configuration cfg = job.getConfiguration();
    cfg.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, totalBytesExpected);
    cfg.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, fileListingPath.toString());
    cfg.setLong(DistCpConstants.CONF_LABEL_TOTAL_NUMBER_OF_RECORDS, totalRecords);
  } catch (NoSuchFieldException | SecurityException | IllegalArgumentException
      | IllegalAccessException | NoSuchMethodException | ClassNotFoundException
      | InvocationTargetException e) {
    throw new IOException(e);
  }
  return fileListingPath;
}
 

@Test
public void testPreserveNothingOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
 

/**
 * If a single file is being copied to a location where the file (of the same
 * name) already exists, then the file shouldn't be skipped.
 */
@Test(timeout=40000)
public void testSingleFileCopy() {
  try {
    deleteState();
    touchFile(SOURCE_PATH + "/1");
    Path sourceFilePath = pathList.get(0);
    Path targetFilePath = new Path(sourceFilePath.toString().replaceAll(
            SOURCE_PATH, TARGET_PATH));
    touchFile(targetFilePath.toString());

    FileSystem fs = cluster.getFileSystem();
    CopyMapper copyMapper = new CopyMapper();
    StubContext stubContext = new StubContext(getConfiguration(), null, 0);
    Mapper<Text, CopyListingFileStatus, Text, Text>.Context context
            = stubContext.getContext();

    context.getConfiguration().set(
            DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH,
            targetFilePath.getParent().toString()); // Parent directory.
    copyMapper.setup(context);

    final CopyListingFileStatus sourceFileStatus = new CopyListingFileStatus(
      fs.getFileStatus(sourceFilePath));

    long before = fs.getFileStatus(targetFilePath).getModificationTime();
    copyMapper.map(new Text(DistCpUtils.getRelativePath(
            new Path(SOURCE_PATH), sourceFilePath)), sourceFileStatus, context);
    long after = fs.getFileStatus(targetFilePath).getModificationTime();

    Assert.assertTrue("File should have been skipped", before == after);

    context.getConfiguration().set(
            DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH,
            targetFilePath.toString()); // Specify the file path.
    copyMapper.setup(context);

    before = fs.getFileStatus(targetFilePath).getModificationTime();
    try { Thread.sleep(2); } catch (Throwable ignore) {}
    copyMapper.map(new Text(DistCpUtils.getRelativePath(
            new Path(SOURCE_PATH), sourceFilePath)), sourceFileStatus, context);
    after = fs.getFileStatus(targetFilePath).getModificationTime();

    Assert.assertTrue("File should have been overwritten.", before < after);

  } catch (Exception exception) {
    Assert.fail("Unexpected exception: " + exception.getMessage());
    exception.printStackTrace();
  }
}
 

@Override
public void recoverTask(TaskAttemptContext context)
    throws IOException {
  if(hasOutputPath()) {
    context.progress();
    TaskAttemptID attemptId = context.getTaskAttemptID();
    int previousAttempt = getAppAttemptId(context) - 1;
    if (previousAttempt < 0) {
      throw new IOException ("Cannot recover task output for first attempt...");
    }

    Path previousCommittedTaskPath = getCommittedTaskPath(
        previousAttempt, context);
    FileSystem fs = previousCommittedTaskPath.getFileSystem(context.getConfiguration());
    if (LOG.isDebugEnabled()) {
      LOG.debug("Trying to recover task from " + previousCommittedTaskPath);
    }
    if (algorithmVersion == 1) {
      if (fs.exists(previousCommittedTaskPath)) {
        Path committedTaskPath = getCommittedTaskPath(context);
        if (fs.exists(committedTaskPath)) {
          if (!fs.delete(committedTaskPath, true)) {
            throw new IOException("Could not delete "+committedTaskPath);
          }
        }
        //Rename can fail if the parent directory does not yet exist.
        Path committedParent = committedTaskPath.getParent();
        fs.mkdirs(committedParent);
        if (!fs.rename(previousCommittedTaskPath, committedTaskPath)) {
          throw new IOException("Could not rename " + previousCommittedTaskPath +
              " to " + committedTaskPath);
        }
      } else {
          LOG.warn(attemptId+" had no output to recover.");
      }
    } else {
      // essentially a no-op, but for backwards compatibility
      // after upgrade to the new fileOutputCommitter,
      // check if there are any output left in committedTaskPath
      if (fs.exists(previousCommittedTaskPath)) {
        LOG.info("Recovering task for upgrading scenario, moving files from "
            + previousCommittedTaskPath + " to " + outputPath);
        FileStatus from = fs.getFileStatus(previousCommittedTaskPath);
        mergePaths(fs, from, outputPath);
      }
      LOG.info("Done recovering task " + attemptId);
    }
  } else {
    LOG.warn("Output Path is null in recoverTask()");
  }
}
 

private FileStatus pathToFileEntry(FileSystem hdfs, String file) 
      throws IOException {
  return hdfs.getFileStatus(new Path(file));
}
 

/**
 * Preserve attribute on file matching that of the file status being sent
 * as argument. Barring the block size, all the other attributes are preserved
 * by this function
 *
 * @param targetFS - File system
 * @param path - Path that needs to preserve original file status
 * @param srcFileStatus - Original file status
 * @param attributes - Attribute set that needs to be preserved
 * @param preserveRawXattrs if true, raw.* xattrs should be preserved
 * @throws IOException - Exception if any (particularly relating to group/owner
 *                       change or any transient error)
 */
public static void preserve(FileSystem targetFS, Path path,
                            CopyListingFileStatus srcFileStatus,
                            EnumSet<FileAttribute> attributes,
                            boolean preserveRawXattrs) throws IOException {

  FileStatus targetFileStatus = targetFS.getFileStatus(path);
  String group = targetFileStatus.getGroup();
  String user = targetFileStatus.getOwner();
  boolean chown = false;

  if (attributes.contains(FileAttribute.ACL)) {
    List<AclEntry> srcAcl = srcFileStatus.getAclEntries();
    List<AclEntry> targetAcl = getAcl(targetFS, targetFileStatus);
    if (!srcAcl.equals(targetAcl)) {
      targetFS.setAcl(path, srcAcl);
    }
    // setAcl doesn't preserve sticky bit, so also call setPermission if needed.
    if (srcFileStatus.getPermission().getStickyBit() !=
        targetFileStatus.getPermission().getStickyBit()) {
      targetFS.setPermission(path, srcFileStatus.getPermission());
    }
  } else if (attributes.contains(FileAttribute.PERMISSION) &&
    !srcFileStatus.getPermission().equals(targetFileStatus.getPermission())) {
    targetFS.setPermission(path, srcFileStatus.getPermission());
  }

  final boolean preserveXAttrs = attributes.contains(FileAttribute.XATTR);
  if (preserveXAttrs || preserveRawXattrs) {
    final String rawNS =
        StringUtils.toLowerCase(XAttr.NameSpace.RAW.name());
    Map<String, byte[]> srcXAttrs = srcFileStatus.getXAttrs();
    Map<String, byte[]> targetXAttrs = getXAttrs(targetFS, path);
    if (srcXAttrs != null && !srcXAttrs.equals(targetXAttrs)) {
      for (Entry<String, byte[]> entry : srcXAttrs.entrySet()) {
        String xattrName = entry.getKey();
        if (xattrName.startsWith(rawNS) || preserveXAttrs) {
          targetFS.setXAttr(path, xattrName, entry.getValue());
        }
      }
    }
  }

  if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDirectory() &&
      (srcFileStatus.getReplication() != targetFileStatus.getReplication())) {
    targetFS.setReplication(path, srcFileStatus.getReplication());
  }

  if (attributes.contains(FileAttribute.GROUP) &&
      !group.equals(srcFileStatus.getGroup())) {
    group = srcFileStatus.getGroup();
    chown = true;
  }

  if (attributes.contains(FileAttribute.USER) &&
      !user.equals(srcFileStatus.getOwner())) {
    user = srcFileStatus.getOwner();
    chown = true;
  }

  if (chown) {
    targetFS.setOwner(path, user, group);
  }
  
  if (attributes.contains(FileAttribute.TIMES)) {
    targetFS.setTimes(path, 
        srcFileStatus.getModificationTime(), 
        srcFileStatus.getAccessTime());
  }
}