Java Code Examples for org.apache.hadoop.hdfs.DFSUtil#getCorruptFiles()

The following examples show how to use org.apache.hadoop.hdfs.DFSUtil#getCorruptFiles() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: LocalBlockIntegrityMonitor.java From RDFS with Apache License 2.0

5 votes

/**
 * @return A list of corrupt files as obtained from the namenode
 */
List<String> getCorruptFiles() throws IOException {
  DistributedFileSystem dfs = helper.getDFS(new Path("/"));

  String[] files = DFSUtil.getCorruptFiles(dfs);
  List<String> corruptFiles = new LinkedList<String>();
  for (String f: files) {
    corruptFiles.add(f);
  }
  RaidUtils.filterTrash(getConf(), corruptFiles);
  return corruptFiles;
}

Example 2

Source File: TestRaidShellFsck.java From RDFS with Apache License 2.0

5 votes

/**
 * sleeps for up to 20s until the number of corrupt files 
 * in the file system is equal to the number specified
 */
static public void waitUntilCorruptFileCount(DistributedFileSystem dfs,
                                       int corruptFiles)
  throws IOException, InterruptedException {
  long waitStart = System.currentTimeMillis();
  while (DFSUtil.getCorruptFiles(dfs).length != corruptFiles &&
      System.currentTimeMillis() < waitStart + 20000L) {
    Thread.sleep(1000);
  }
  assertEquals("expected " + corruptFiles + " corrupt files", 
      corruptFiles, DFSUtil.getCorruptFiles(dfs).length);
}

Example 3

Source File: TestRaidShellFsck_CorruptCounter.java From RDFS with Apache License 2.0

5 votes

/**
 * sleeps for up to 20s until the number of corrupt files 
 * in the file system is equal to the number specified
 */
private void waitUntilCorruptFileCount(DistributedFileSystem dfs,
                                       int corruptFiles)
  throws IOException {
  int initialCorruptFiles = DFSUtil.getCorruptFiles(dfs).length;
  long waitStart = System.currentTimeMillis();
  while (DFSUtil.getCorruptFiles(dfs).length != corruptFiles) {
    try {
      Thread.sleep(1000);
    } catch (InterruptedException ignore) {
      
    }

    if (System.currentTimeMillis() > waitStart + 20000L) {
      break;
    }
  }
  
  long waited = System.currentTimeMillis() - waitStart;

  int corruptFilesFound = DFSUtil.getCorruptFiles(dfs).length;
  if (corruptFilesFound != corruptFiles) {
    throw new IOException("expected " + corruptFiles + 
                          " corrupt files but got " +
                          corruptFilesFound);
  }
}

Example 4

Source File: FileFixer.java From RDFS with Apache License 2.0

5 votes

/**
 * @return A list of corrupt files as obtained from the namenode
 * If the namenode is down, then return an empty list.
 */
List<Path> getCorruptFilesFromNamenode(FileSystem fs) throws IOException {
  if (!(fs instanceof DistributedFileSystem)) {
    throw new IOException("Only DistributedFileSystem can be handled " +
                          " by HighTide.");
  }

  DistributedFileSystem dfs = (DistributedFileSystem) fs;
  List<Path> corruptFiles = new LinkedList<Path>();

  try {
    LOG.info("Checking filesystem: " + dfs.getUri());
    String[] files = 
      DFSUtil.getCorruptFiles(dfs);
    for (String f: files) {
      Path p = new Path(f).makeQualified(fs);
      corruptFiles.add(p);
    }
    return corruptFiles;
  } catch (Exception e) {
    // if a single namenode is down, log it and ignore. Continue to
    // fix other namenodes.
    LOG.warn("getCorruptFilesFromNamenode: Unable to contact filesystem: " + fs.getUri() +
             " ignoring..." + e);
    e.printStackTrace();
    return corruptFiles;
  }
}

Example 5

Source File: TestFileCorruption.java From RDFS with Apache License 2.0

4 votes

/**
 * check if listCorruptFileBlocks() returns the right number of
 * corrupt files if there are two corrupt files with the same name
 * in different directories
 */
public void test2CorruptFilesWithSameName() throws Exception {
  MiniDFSCluster cluster = null;
  Random random = new Random();
  
  try {
    Configuration conf = new Configuration();
    // datanode scans directories
    conf.setInt("dfs.datanode.directoryscan.interval", 1);
    // datanode sends block reports 
    conf.setInt("dfs.blockreport.intervalMsec", 3 * 1000);
    conf.setBoolean("dfs.permissions", false);
    cluster = new MiniDFSCluster(conf, 1, true, null);
    FileSystem fs = cluster.getFileSystem();
    
    assertTrue("fs is not a DFS", fs instanceof DistributedFileSystem);
    DistributedFileSystem dfs = (DistributedFileSystem) fs;

    Path file1 = new Path("/srcdat12/test2file.test");
    Path file2 = new Path("/srcdat13/test2file.test");
    // create two files with the same name
    DFSTestUtil.createFile(fs, file1, 1L, (short)1, 1L);
    DFSTestUtil.createFile(fs, file2, 1L, (short)1, 1L);

    // fetch bad file list from namenode. There should be none.
    ClientProtocol namenode = DFSClient.createNamenode(conf);
    String[] badFiles = DFSUtil.getCorruptFiles(dfs);
    assertTrue("Namenode has " + badFiles.length +
               " corrupt files. Expecting None.",
        badFiles.length == 0);

    // Now deliberately corrupt one block in each file
    Path[] files = {file1, file2};
    for (Path file: files) {
      LocatedBlocks fileBlocks = 
        namenode.getBlockLocations(file.toString(), 0, 1L);
      LocatedBlock block = fileBlocks.get(0);
      File data_dir = 
        new File(TEST_ROOT_DIR, "dfs/data/");
      File dir1 = cluster.getBlockDirectory("data"+(2 * 0 + 1));
      File dir2 = cluster.getBlockDirectory("data"+(2 * 0 + 2));
      if (!(dir1.isDirectory() && dir2.isDirectory())) {
        throw new IOException("data directories not found for data node 0: " +
                              dir1.toString() + " " + dir2.toString());
      }

      File[] dirs = new File[2];
      dirs[0] = dir1; 
      dirs[1] = dir2;
      for (File dir: dirs) {
        File[] blockFiles = dir.listFiles();
        if ((blockFiles == null) || (blockFiles.length == 0)) {
          throw 
            new IOException("no blocks found in data node's data directory");
        }

        for (File blockFile: blockFiles) {
          if ((blockFile.getName().
               startsWith("blk_" + block.getBlock().getBlockId())) &&
              (!blockFile.getName().endsWith(".meta"))) {
            blockFile.delete();
          }
        }
      }
      LocatedBlock[] toReport = { block };
      namenode.reportBadBlocks(toReport);
    }

    // fetch bad file list from namenode. There should be 2.
    badFiles = DFSUtil.getCorruptFiles(dfs);
    assertTrue("Namenode has " + badFiles.length + " bad files. Expecting 2.",
        badFiles.length == 2);
  } finally {
    if (cluster != null) {
      cluster.shutdown(); 
    }
  }
}

Example 6