Java Code Examples for org.apache.hadoop.fs.FileUtil#listFiles()

The following examples show how to use org.apache.hadoop.fs.FileUtil#listFiles() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Storage.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * @return true if the storage directory should prompt the user prior
 * to formatting (i.e if the directory appears to contain some data)
 * @throws IOException if the SD cannot be accessed due to an IO error
 */
@Override
public boolean hasSomeData() throws IOException {
  // Its alright for a dir not to exist, or to exist (properly accessible)
  // and be completely empty.
  if (!root.exists()) return false;
  
  if (!root.isDirectory()) {
    // a file where you expect a directory should not cause silent
    // formatting
    return true;
  }
  
  if (FileUtil.listFiles(root).length == 0) {
    // Empty dir can format without prompt.
    return false;
  }
  
  return true;
}
Example 2
Source File: JNStorage.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Purge files in the given directory which match any of the set of patterns.
 * The patterns must have a single numeric capture group which determines
 * the associated transaction ID of the file. Only those files for which
 * the transaction ID is less than the <code>minTxIdToKeep</code> parameter
 * are removed.
 */
private static void purgeMatching(File dir, List<Pattern> patterns,
    long minTxIdToKeep) throws IOException {

  for (File f : FileUtil.listFiles(dir)) {
    if (!f.isFile()) continue;
    
    for (Pattern p : patterns) {
      Matcher matcher = p.matcher(f.getName());
      if (matcher.matches()) {
        // This parsing will always succeed since the group(1) is
        // /\d+/ in the regex itself.
        long txid = Long.parseLong(matcher.group(1));
        if (txid < minTxIdToKeep) {
          LOG.info("Purging no-longer needed file " + txid);
          if (!f.delete()) {
            LOG.warn("Unable to delete no-longer-needed data " +
                f);
          }
          break;
        }
      }
    }
  }
}
Example 3
Source File: GenericTestUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * List all of the files in 'dir' that match the regex 'pattern'.
 * Then check that this list is identical to 'expectedMatches'.
 * @throws IOException if the dir is inaccessible
 */
public static void assertGlobEquals(File dir, String pattern,
    String ... expectedMatches) throws IOException {
  
  Set<String> found = Sets.newTreeSet();
  for (File f : FileUtil.listFiles(dir)) {
    if (f.getName().matches(pattern)) {
      found.add(f.getName());
    }
  }
  Set<String> expectedSet = Sets.newTreeSet(
      Arrays.asList(expectedMatches));
  Assert.assertEquals("Bad files matching " + pattern + " in " + dir,
      Joiner.on(",").join(expectedSet),
      Joiner.on(",").join(found));
}
Example 4
Source File: Storage.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * @return true if the storage directory should prompt the user prior
 * to formatting (i.e if the directory appears to contain some data)
 * @throws IOException if the SD cannot be accessed due to an IO error
 */
@Override
public boolean hasSomeData() throws IOException {
  // Its alright for a dir not to exist, or to exist (properly accessible)
  // and be completely empty.
  if (!root.exists()) return false;
  
  if (!root.isDirectory()) {
    // a file where you expect a directory should not cause silent
    // formatting
    return true;
  }
  
  if (FileUtil.listFiles(root).length == 0) {
    // Empty dir can format without prompt.
    return false;
  }
  
  return true;
}
Example 5
Source File: JNStorage.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Purge files in the given directory which match any of the set of patterns.
 * The patterns must have a single numeric capture group which determines
 * the associated transaction ID of the file. Only those files for which
 * the transaction ID is less than the <code>minTxIdToKeep</code> parameter
 * are removed.
 */
private static void purgeMatching(File dir, List<Pattern> patterns,
    long minTxIdToKeep) throws IOException {

  for (File f : FileUtil.listFiles(dir)) {
    if (!f.isFile()) continue;
    
    for (Pattern p : patterns) {
      Matcher matcher = p.matcher(f.getName());
      if (matcher.matches()) {
        // This parsing will always succeed since the group(1) is
        // /\d+/ in the regex itself.
        long txid = Long.parseLong(matcher.group(1));
        if (txid < minTxIdToKeep) {
          LOG.info("Purging no-longer needed file " + txid);
          if (!f.delete()) {
            LOG.warn("Unable to delete no-longer-needed data " +
                f);
          }
          break;
        }
      }
    }
  }
}
Example 6
Source File: GenericTestUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * List all of the files in 'dir' that match the regex 'pattern'.
 * Then check that this list is identical to 'expectedMatches'.
 * @throws IOException if the dir is inaccessible
 */
public static void assertGlobEquals(File dir, String pattern,
    String ... expectedMatches) throws IOException {
  
  Set<String> found = Sets.newTreeSet();
  for (File f : FileUtil.listFiles(dir)) {
    if (f.getName().matches(pattern)) {
      found.add(f.getName());
    }
  }
  Set<String> expectedSet = Sets.newTreeSet(
      Arrays.asList(expectedMatches));
  Assert.assertEquals("Bad files matching " + pattern + " in " + dir,
      Joiner.on(",").join(expectedSet),
      Joiner.on(",").join(found));
}
Example 7
Source File: FileJournalManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public void purgeLogsOlderThan(long minTxIdToKeep)
    throws IOException {
  LOG.info("Purging logs older than " + minTxIdToKeep);
  File[] files = FileUtil.listFiles(sd.getCurrentDir());
  List<EditLogFile> editLogs = matchEditLogs(files, true);
  for (EditLogFile log : editLogs) {
    if (log.getFirstTxId() < minTxIdToKeep &&
        log.getLastTxId() < minTxIdToKeep) {
      purger.purgeLog(log);
    }
  }
}
Example 8
Source File: FileJournalManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public void purgeLogsOlderThan(long minTxIdToKeep)
    throws IOException {
  LOG.info("Purging logs older than " + minTxIdToKeep);
  File[] files = FileUtil.listFiles(sd.getCurrentDir());
  List<EditLogFile> editLogs = matchEditLogs(files, true);
  for (EditLogFile log : editLogs) {
    if (log.getFirstTxId() < minTxIdToKeep &&
        log.getLastTxId() < minTxIdToKeep) {
      purger.purgeLog(log);
    }
  }
}
Example 9
Source File: WindowsSecureContainerExecutor.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public boolean delete(Path p, boolean recursive) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug(String.format("EFS:delete: %s %b", p, recursive));
  }
  
  // The super delete uses the FileUtil.fullyDelete, 
  // but we cannot rely on that because we need to use the elevated 
  // operations to remove the files
  //
  File f = pathToFile(p);
  if (!f.exists()) {
    //no path, return false "nothing to delete"
    return false;
  }
  else if (f.isFile()) {
    return Native.Elevated.deleteFile(p);
  } 
  else if (f.isDirectory()) {
    
    // This is a best-effort attempt. There are race conditions in that
    // child files can be created/deleted after we snapped the list. 
    // No need to protect against that case.
    File[] files = FileUtil.listFiles(f);
    int childCount = files.length;
    
    if (recursive) {
      for(File child:files) {
        if (delete(new Path(child.getPath()), recursive)) {
          --childCount;
        }
      }
    }
    if (childCount == 0) {
      return Native.Elevated.deleteDirectory(p);
    } 
    else {
      throw new IOException("Directory " + f.toString() + " is not empty");
    }
  }
  else {
    // This can happen under race conditions if an external agent 
    // is messing with the file type between IFs
    throw new IOException("Path " + f.toString() + 
        " exists, but is neither a file nor a directory");
  }
}
Example 10
Source File: WindowsSecureContainerExecutor.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public boolean delete(Path p, boolean recursive) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug(String.format("EFS:delete: %s %b", p, recursive));
  }
  
  // The super delete uses the FileUtil.fullyDelete, 
  // but we cannot rely on that because we need to use the elevated 
  // operations to remove the files
  //
  File f = pathToFile(p);
  if (!f.exists()) {
    //no path, return false "nothing to delete"
    return false;
  }
  else if (f.isFile()) {
    return Native.Elevated.deleteFile(p);
  } 
  else if (f.isDirectory()) {
    
    // This is a best-effort attempt. There are race conditions in that
    // child files can be created/deleted after we snapped the list. 
    // No need to protect against that case.
    File[] files = FileUtil.listFiles(f);
    int childCount = files.length;
    
    if (recursive) {
      for(File child:files) {
        if (delete(new Path(child.getPath()), recursive)) {
          --childCount;
        }
      }
    }
    if (childCount == 0) {
      return Native.Elevated.deleteDirectory(p);
    } 
    else {
      throw new IOException("Directory " + f.toString() + " is not empty");
    }
  }
  else {
    // This can happen under race conditions if an external agent 
    // is messing with the file type between IFs
    throw new IOException("Path " + f.toString() + 
        " exists, but is neither a file nor a directory");
  }
}