Java Code Examples for org.apache.hadoop.fs.FileStatus#isDirectory()

private void recursiveList(FileSystem fs, Path p, String id, ArrayList<FileStatus> results) throws IOException {
    if (log.isDebugEnabled())
        log.debug("Checking path: " + p.getName());
    FileStatus[] list = fs.listStatus(p);
    if (null != list && list.length > 0) {
        for (FileStatus stat : list) {
            if (stat.isDirectory()) {
                log.debug(stat.getPath().getName() + " is a directory");
                recursiveList(fs, stat.getPath(), id, results);
            } else {
                log.debug(stat.getPath().getName() + " is not a directory");
                if (stat.getPath().getName().equals(id + ".alias") || stat.getPath().getName().equals(id + ".view")
                                || stat.getPath().getName().equals(id + ".queryId")) {
                    results.add(stat);
                } else {
                    log.debug(stat.getPath().getName() + " does not match filter");
                }
            }
        }
    }
}
 

public static boolean checkIfFoldersAreInSync(FileSystem fs, String targetBase, String sourceBase)
    throws IOException {
  Path base = new Path(targetBase);

   Stack<Path> stack = new Stack<Path>();
   stack.push(base);
   while (!stack.isEmpty()) {
     Path file = stack.pop();
     if (!fs.exists(file)) continue;
     FileStatus[] fStatus = fs.listStatus(file);
     if (fStatus == null || fStatus.length == 0) continue;

     for (FileStatus status : fStatus) {
       if (status.isDirectory()) {
         stack.push(status.getPath());
       }
       Assert.assertTrue(fs.exists(new Path(sourceBase + "/" +
           DistCpUtils.getRelativePath(new Path(targetBase), status.getPath()))));
     }
   }
   return true;
}
 

protected BigInteger checkContents(FileStatus fileStatus, FileSystem fileSystem) throws IOException {
  if (fileStatus.isDirectory()) {
    LOG.debug("Scanning directory [{0}].", fileStatus.getPath());
    BigInteger count = BigInteger.ZERO;
    Path path = fileStatus.getPath();
    FileStatus[] listStatus = fileSystem.listStatus(path);
    for (FileStatus fs : listStatus) {
      count = count.add(checkContents(fs, fileSystem));
    }
    return count;
  } else {
    int hashCode = fileStatus.getPath().toString().hashCode();
    long modificationTime = fileStatus.getModificationTime();
    long len = fileStatus.getLen();
    BigInteger bi = BigInteger.valueOf(hashCode).add(
        BigInteger.valueOf(modificationTime).add(BigInteger.valueOf(len)));
    LOG.debug("File path hashcode [{0}], mod time [{1}], len [{2}] equals file code [{3}].",
        Integer.toString(hashCode), Long.toString(modificationTime), Long.toString(len),
        bi.toString(Character.MAX_RADIX));
    return bi;
  }
}
 

public File(String schemaName, WorkspaceSchemaFactory.WorkspaceSchema wsSchema, FileStatus fileStatus) {
  this.SCHEMA_NAME = schemaName;
  this.ROOT_SCHEMA_NAME = wsSchema.getSchemaPath().get(0);
  this.WORKSPACE_NAME = wsSchema.getName();
  this.FILE_NAME = fileStatus.getPath().getName();
  this.RELATIVE_PATH = Path.getPathWithoutSchemeAndAuthority(new Path(wsSchema.getDefaultLocation())).toUri()
    .relativize(Path.getPathWithoutSchemeAndAuthority(fileStatus.getPath()).toUri()).getPath();
  this.IS_DIRECTORY = fileStatus.isDirectory();
  this.IS_FILE = fileStatus.isFile();
  this.LENGTH = fileStatus.getLen();
  this.OWNER = fileStatus.getOwner();
  this.GROUP = fileStatus.getGroup();
  this.PERMISSION = fileStatus.getPermission().toString();
  this.ACCESS_TIME = getTimestampWithReplacedZone(fileStatus.getAccessTime());
  this.MODIFICATION_TIME = getTimestampWithReplacedZone(fileStatus.getModificationTime());
}
 

/**
 * Read the given path as a Java RDD; The path can have second level sub folder.
 * @param inputPath
 * @param fs
 * @param sc
 * @param keyClass
 * @param valueClass
 * @return
 * @throws IOException
 */
public static JavaPairRDD parseInputPath(String inputPath, FileSystem fs, JavaSparkContext sc, Class keyClass,
        Class valueClass) throws IOException {
    List<String> inputFolders = Lists.newArrayList();
    Path inputHDFSPath = new Path(inputPath);
    FileStatus[] fileStatuses = fs.listStatus(inputHDFSPath);
    boolean hasDir = false;
    for (FileStatus stat : fileStatuses) {
        if (stat.isDirectory() && !stat.getPath().getName().startsWith("_")) {
            hasDir = true;
            inputFolders.add(stat.getPath().toString());
        }
    }

    if (!hasDir) {
        return sc.sequenceFile(inputHDFSPath.toString(), keyClass, valueClass);
    }

    return sc.sequenceFile(StringUtil.join(inputFolders, ","), keyClass, valueClass);
}
 

/**
 * Scans the directory for new files.
 */
protected void scanDirectory()
{
  if (System.currentTimeMillis() - scanIntervalMillis >= lastScanMillis) {
    Set<Path> newPaths = scanner.scan(fs, filePath, processedFiles);

    for (Path newPath : newPaths) {
      try {
        FileStatus fileStatus = fs.getFileStatus(newPath);
        if (fileStatus.isDirectory())  {
          checkVisitedDirectory(newPath);
        } else {
          String newPathString = newPath.toString();
          pendingFiles.add(newPathString);
          processedFiles.add(newPathString);
          localProcessedFileCount.increment();
        }
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }

    lastScanMillis = System.currentTimeMillis();
  }
}
 

private static List<Path> copyFiles(FileSystem srcFS, Path src, FileSystem dstFS, Path dst,
    Configuration conf, ExecutorService pool, List<Future<Void>> futures) throws IOException {
  List<Path> traversedPaths = new ArrayList<>();
  traversedPaths.add(dst);
  FileStatus currentFileStatus = srcFS.getFileStatus(src);
  if (currentFileStatus.isDirectory()) {
    if (!dstFS.mkdirs(dst)) {
      throw new IOException("Create directory failed: " + dst);
    }
    FileStatus[] subPaths = srcFS.listStatus(src);
    for (FileStatus subPath : subPaths) {
      traversedPaths.addAll(copyFiles(srcFS, subPath.getPath(), dstFS,
        new Path(dst, subPath.getPath().getName()), conf, pool, futures));
    }
  } else {
    Future<Void> future = pool.submit(() -> {
      FileUtil.copy(srcFS, src, dstFS, dst, false, false, conf);
      return null;
    });
    futures.add(future);
  }
  return traversedPaths;
}
 

public void addDirectory(WrappedFile dirPath, List<WrappedFile> directories) throws IOException {
  PathFilter pathFilter = new PathFilter() {
    @Override
    public boolean accept(Path entry) {
      try {
        FileStatus fileStatus = fs.getFileStatus(entry);
        if (fileStatus.isDirectory()) {
          if (processSubdirectories) {
            directories.add(new HdfsFile(fs, entry));
            addDirectory(getFile(entry.toString()), directories);
          }
          return false;
        }
      } catch (IOException ex) {
        LOG.error("Failed to open file {}", entry.toString(), ex);
      }
      return false;
    }
  };

  fs.globStatus(new Path(dirPath.getAbsolutePath(), "*"), pathFilter);

  if (!directories.contains(dirPath)) {
    directories.add(dirPath);
  }
}
 

public boolean containsDirectories(DrillFileSystem fs) throws IOException {
  if (dirStatus == StatusType.NOT_CHECKED) {
    dirStatus = StatusType.NO_DIRS;
    for (FileStatus status : getStatuses(fs)) {
      if (status.isDirectory()) {
        dirStatus = StatusType.HAS_DIRS;
        break;
      }
    }
  }
  return dirStatus == StatusType.HAS_DIRS;
}
 

private void verifyPermsRecursively(FileSystem fs,
    FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  FileStatus status = files.getFileStatus(p);
  if (status.isDirectory()) {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_DIR_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_DIR_PERMS.toShort());
    }
    if (!status.isSymlink()) {
      FileStatus[] statuses = fs.listStatus(p);
      for (FileStatus stat : statuses) {
        verifyPermsRecursively(fs, files, stat.getPath(), vis);
      }
    }
  }
  else {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_FILE_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_FILE_PERMS.toShort());
    }
  }      
}
 

private void cleanupExtraFileFromSpecX(FileSystem fileSystem, String uuid, Path fileCache) throws IOException {
  FileStatus[] listStatus = fileSystem.listStatus(fileCache);
  List<FileStatus> uuidPaths = new ArrayList<FileStatus>();
  for (FileStatus fs : listStatus) {
    Path path = fs.getPath();
    if (fs.isDirectory()) {
      cleanupExtraFileFromSpecX(fileSystem, uuid, path);
    } else if (path.getName().startsWith(uuid)) {
      uuidPaths.add(fs);
    }
  }
  if (uuidPaths.size() > 1) {
    deleteIncomplete(fileSystem, uuidPaths);
  }
}
 

private static boolean isDirectoryAndNotEmpty(FileSystem fileSystem, FileStatus fileStatus) throws IOException {
  return fileStatus.isDirectory() && getChildren(fileSystem, fileStatus).length > 0;
}
 

void moveFile(byte[]src, byte[] dst, CopyOption... options)
       throws IOException
   {
checkWritable();
      if (Arrays.equals(src, dst))
          return;    // do nothing, src and dst are the same

      beginWrite();
      try {
          ensureOpen();
          org.apache.hadoop.fs.Path eSrc_path = new HadoopPath(this, src).getRawResolvedPath();
          FileStatus eSrc = this.fs.getFileStatus(eSrc_path);
          if (!this.fs.exists(eSrc_path))
              throw new NoSuchFileException(getString(src));
          if (eSrc.isDirectory()) {    // specification says to create dst directory
              createDirectory(dst);
              return;
          }
          boolean hasReplace = false;
          boolean hasCopyAttrs = false;
          for (CopyOption opt : options) {
              if (opt == REPLACE_EXISTING)
                  hasReplace = true;
              else if (opt == COPY_ATTRIBUTES)
                  hasCopyAttrs = true;
          }
          org.apache.hadoop.fs.Path eDst_path = new HadoopPath(this, dst).getRawResolvedPath();
  
          if (fs.exists(eDst_path)) {
              if (!hasReplace)
                  throw new FileAlreadyExistsException(getString(dst));

              if(!fs.delete(eDst_path, false)) {
              	throw new AccessDeniedException("cannot delete hdfs file " + getString(dst));
              }
          }
          //Simply rename the path
          if (!fs.rename(eSrc_path, eDst_path)) {
          	throw new AccessDeniedException("cannot move source file " + eSrc_path.toString());
          }
 
      } finally {
          endWrite();
      }
  }
 

/**
 * Processes the input file/folder argument. If the input is a file,
 * then it is directly considered for further processing by TraceBuilder.
 * If the input is a folder, then all the history logs in the
 * input folder are considered for further processing.
 *
 * If isRecursive is true, then the input path is recursively scanned
 * for job history logs for further processing by TraceBuilder.
 *
 * NOTE: If the input represents a globbed path, then it is first flattened
 *       and then the individual paths represented by the globbed input
 *       path are considered for further processing.
 *
 * @param input        input path, possibly globbed
 * @param conf         configuration
 * @param isRecursive  whether to recursively traverse the input paths to
 *                     find history logs
 * @return the input history log files' paths
 * @throws FileNotFoundException
 * @throws IOException
 */
static List<Path> processInputArgument(String input, Configuration conf,
    boolean isRecursive) throws FileNotFoundException, IOException {
  Path inPath = new Path(input);
  FileSystem fs = inPath.getFileSystem(conf);
  FileStatus[] inStatuses = fs.globStatus(inPath);

  List<Path> inputPaths = new LinkedList<Path>();
  if (inStatuses == null || inStatuses.length == 0) {
    return inputPaths;
  }

  for (FileStatus inStatus : inStatuses) {
    Path thisPath = inStatus.getPath();
    if (inStatus.isDirectory()) {

      // Find list of files in this path(recursively if -recursive option
      // is specified).
      List<FileStatus> historyLogs = new ArrayList<FileStatus>();

      RemoteIterator<LocatedFileStatus> iter =
        fs.listFiles(thisPath, isRecursive);
      while (iter.hasNext()) {
        LocatedFileStatus child = iter.next();
        String fileName = child.getPath().getName();

        if (!(fileName.endsWith(".crc") || fileName.startsWith("."))) {
          historyLogs.add(child);
        }
      }

      if (historyLogs.size() > 0) {
        // Add the sorted history log file names in this path to the
        // inputPaths list
        FileStatus[] sortableNames =
            historyLogs.toArray(new FileStatus[historyLogs.size()]);
        Arrays.sort(sortableNames, new HistoryLogsComparator());

        for (FileStatus historyLog : sortableNames) {
          inputPaths.add(historyLog.getPath());
        }
      }
    } else {
      inputPaths.add(thisPath);
    }
  }

  return inputPaths;
}
 

@Override
public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(OPTION_INPUT_PATH);
    options.addOption(OPTION_HTABLE_NAME);
    options.addOption(OPTION_CUBE_NAME);
    parseOptions(options, args);

    String tableName = getOptionValue(OPTION_HTABLE_NAME);
    // e.g
    // /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/
    // end with "/"
    String input = getOptionValue(OPTION_INPUT_PATH);

    Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
    FsShell shell = new FsShell(conf);

    int exitCode = -1;
    int retryCount = 10;
    while (exitCode != 0 && retryCount >= 1) {
        exitCode = shell.run(new String[] { "-chmod", "-R", "777", input });
        retryCount--;
        Thread.sleep(5000);
    }

    if (exitCode != 0) {
        logger.error("Failed to change the file permissions: " + input);
        throw new IOException("Failed to change the file permissions: " + input);
    }

    String[] newArgs = new String[2];
    newArgs[0] = input;
    newArgs[1] = tableName;

    int count = 0;
    Path inputPath = new Path(input);
    FileSystem fs = HadoopUtil.getFileSystem(inputPath);
    FileStatus[] fileStatuses = fs.listStatus(inputPath);

    for (FileStatus fileStatus : fileStatuses) {
        if (fileStatus.isDirectory()) {
            Path path = fileStatus.getPath();
            if (path.getName().equals(FileOutputCommitter.TEMP_DIR_NAME)) {
                logger.info("Delete temporary path: " + path);
                fs.delete(path, true);
            } else {
                count++;
            }
        }
    }

    int ret = 0;
    if (count > 0) {
        logger.debug("Start to run LoadIncrementalHFiles");
        ret = MRUtil.runMRJob(new LoadIncrementalHFiles(conf), newArgs);
        logger.debug("End to run LoadIncrementalHFiles");
        return ret;
    } else {
        logger.debug("Nothing to load, cube is empty");
        return ret;
    }
}
 

public int run(String [] argv) throws Exception {
  JobConf job = new JobConf(getConf());
  job.setJarByClass(GenericMRLoadGenerator.class);
  job.setMapperClass(SampleMapper.class);
  job.setReducerClass(SampleReducer.class);
  if (!parseArgs(argv, job)) {
    return -1;
  }

  if (null == FileOutputFormat.getOutputPath(job)) {
    // No output dir? No writes
    job.setOutputFormat(NullOutputFormat.class);
  }

  if (0 == FileInputFormat.getInputPaths(job).length) {
    // No input dir? Generate random data
    System.err.println("No input path; ignoring InputFormat");
    confRandom(job);
  } else if (null != job.getClass(
     org.apache.hadoop.mapreduce.GenericMRLoadGenerator.INDIRECT_INPUT_FORMAT,
     null)) {
    // specified IndirectInputFormat? Build src list
    JobClient jClient = new JobClient(job);
    Path tmpDir = new Path(jClient.getFs().getHomeDirectory(), ".staging");
    Random r = new Random();
    Path indirInputFile = new Path(tmpDir,
        Integer.toString(r.nextInt(Integer.MAX_VALUE), 36) + "_files");
    job.set(
      org.apache.hadoop.mapreduce.GenericMRLoadGenerator.INDIRECT_INPUT_FILE,
      indirInputFile.toString());
    SequenceFile.Writer writer = SequenceFile.createWriter(
        tmpDir.getFileSystem(job), job, indirInputFile,
        LongWritable.class, Text.class,
        SequenceFile.CompressionType.NONE);
    try {
      for (Path p : FileInputFormat.getInputPaths(job)) {
        FileSystem fs = p.getFileSystem(job);
        Stack<Path> pathstack = new Stack<Path>();
        pathstack.push(p);
        while (!pathstack.empty()) {
          for (FileStatus stat : fs.listStatus(pathstack.pop())) {
            if (stat.isDirectory()) {
              if (!stat.getPath().getName().startsWith("_")) {
                pathstack.push(stat.getPath());
              }
            } else {
              writer.sync();
              writer.append(new LongWritable(stat.getLen()),
                  new Text(stat.getPath().toUri().toString()));
            }
          }
        }
      }
    } finally {
      writer.close();
    }
  }

  Date startTime = new Date();
  System.out.println("Job started: " + startTime);
  JobClient.runJob(job);
  Date endTime = new Date();
  System.out.println("Job ended: " + endTime);
  System.out.println("The job took " +
                     (endTime.getTime() - startTime.getTime()) /1000 +
                     " seconds.");

  return 0;
}
 

@Override
public boolean isFileDeletable(FileStatus fStat) {
  try {
    // if its a directory, then it can be deleted
    if (fStat.isDirectory()) {
      return true;
    }

    Path file = fStat.getPath();

    // we need the table and region to determine if this is from a mob region
    // we don't need to worry about hfilelink back references, because the hfilelink cleaner will
    // retain them.
    Path family = file.getParent();
    Path region = family.getParent();
    Path table = region.getParent();

    TableName tableName = CommonFSUtils.getTableName(table);

    String mobRegion = MOB_REGIONS.get(tableName);
    if (mobRegion == null) {
      String tmp = MobUtils.getMobRegionInfo(tableName).getEncodedName();
      if (tmp == null) {
        LOG.error("couldn't determine mob region for table {} keeping files just in case.",
            tableName);
        return false;
      }
      mobRegion = MOB_REGIONS.putIfAbsent(tableName, tmp);
      // a return of null means that tmp is now in the map for future lookups.
      if (mobRegion == null) {
        mobRegion = tmp;
      }
      LOG.debug("Had to calculate name of mob region for table {} and it is {}", tableName,
          mobRegion);
    }

    boolean ret = !mobRegion.equals(region.getName());
    if (LOG.isDebugEnabled() && !ret) {
      LOG.debug("Keeping file '{}' because it is from mob dir", fStat.getPath());
    }
    return ret;
  } catch (RuntimeException e) {
    LOG.error("Failed to determine mob status of '{}', keeping it just in case.", fStat.getPath(),
        e);
    return false;
  }
}
 

/**
 * Implementation of the Mapper::map(). Does the copy.
 * @param relPath The target path.
 * @param sourceFileStatus The source path.
 * @throws IOException
 * @throws InterruptedException
 */
@Override
public void map(Text relPath, CopyListingFileStatus sourceFileStatus,
        Context context) throws IOException, InterruptedException {
  Path sourcePath = sourceFileStatus.getPath();

  if (LOG.isDebugEnabled())
    LOG.debug("DistCpMapper::map(): Received " + sourcePath + ", " + relPath);

  Path target = new Path(targetWorkPath.makeQualified(targetFS.getUri(),
                        targetFS.getWorkingDirectory()) + relPath.toString());

  EnumSet<DistCpOptions.FileAttribute> fileAttributes
          = getFileAttributeSettings(context);
  final boolean preserveRawXattrs = context.getConfiguration().getBoolean(
      DistCpConstants.CONF_LABEL_PRESERVE_RAWXATTRS, false);

  final String description = "Copying " + sourcePath + " to " + target;
  context.setStatus(description);

  LOG.info(description);

  try {
    CopyListingFileStatus sourceCurrStatus;
    FileSystem sourceFS;
    try {
      sourceFS = sourcePath.getFileSystem(conf);
      final boolean preserveXAttrs =
          fileAttributes.contains(FileAttribute.XATTR);
      sourceCurrStatus = DistCpUtils.toCopyListingFileStatus(sourceFS,
        sourceFS.getFileStatus(sourcePath),
        fileAttributes.contains(FileAttribute.ACL), 
        preserveXAttrs, preserveRawXattrs);
    } catch (FileNotFoundException e) {
      throw new IOException(new RetriableFileCopyCommand.CopyReadException(e));
    }

    FileStatus targetStatus = null;

    try {
      targetStatus = targetFS.getFileStatus(target);
    } catch (FileNotFoundException ignore) {
      if (LOG.isDebugEnabled())
        LOG.debug("Path could not be found: " + target, ignore);
    }

    if (targetStatus != null && (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) {
      throw new IOException("Can't replace " + target + ". Target is " +
          getFileType(targetStatus) + ", Source is " + getFileType(sourceCurrStatus));
    }

    if (sourceCurrStatus.isDirectory()) {
      createTargetDirsWithRetry(description, target, context);
      return;
    }

    FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target);
    if (action == FileAction.SKIP) {
      LOG.info("Skipping copy of " + sourceCurrStatus.getPath()
               + " to " + target);
      updateSkipCounters(context, sourceCurrStatus);
      context.write(null, new Text("SKIP: " + sourceCurrStatus.getPath()));
    } else {
      copyFileWithRetry(description, sourceCurrStatus, target, context,
          action, fileAttributes);
    }

    DistCpUtils.preserve(target.getFileSystem(conf), target, sourceCurrStatus,
        fileAttributes, preserveRawXattrs);
  } catch (IOException exception) {
    handleFailures(exception, sourceFileStatus, target, context);
  }
}
 

/**
 * Merge two paths together.  Anything in from will be moved into to, if there
 * are any name conflicts while merging the files or directories in from win.
 * @param fs the File System to use
 * @param from the path data is coming from.
 * @param to the path data is going to.
 * @throws IOException on any error
 */
private void mergePaths(FileSystem fs, final FileStatus from,
    final Path to) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Merging data from " + from + " to " + to);
  }
  FileStatus toStat;
  try {
    toStat = fs.getFileStatus(to);
  } catch (FileNotFoundException fnfe) {
    toStat = null;
  }

  if (from.isFile()) {
    if (toStat != null) {
      if (!fs.delete(to, true)) {
        throw new IOException("Failed to delete " + to);
      }
    }

    if (!fs.rename(from.getPath(), to)) {
      throw new IOException("Failed to rename " + from + " to " + to);
    }
  } else if (from.isDirectory()) {
    if (toStat != null) {
      if (!toStat.isDirectory()) {
        if (!fs.delete(to, true)) {
          throw new IOException("Failed to delete " + to);
        }
        renameOrMerge(fs, from, to);
      } else {
        //It is a directory so merge everything in the directories
        for (FileStatus subFrom : fs.listStatus(from.getPath())) {
          Path subTo = new Path(to, subFrom.getPath().getName());
          mergePaths(fs, subFrom, subTo);
        }
      }
    } else {
      renameOrMerge(fs, from, to);
    }
  }
}
 

/**
 * This test contains the is-directory logic for Swift, so if
 * changed there is only one place for it.
 *
 * @param fileStatus status to examine
 * @return true if we consider this status to be representative of a
 *         directory.
 */
public static boolean isDirectory(FileStatus fileStatus) {
  return fileStatus.isDirectory() || isFilePretendingToBeDirectory(fileStatus);
}