Java Code Examples for org.apache.hadoop.hdfs.DFSTestUtil#readFile()

@Test(timeout = 30000)
public void testReadRenamedSnapshotFileWithCheckpoint() throws Exception {
  final Path foo = new Path("/foo");
  final Path foo2 = new Path("/foo2");
  hdfs.mkdirs(foo);
  hdfs.mkdirs(foo2);

  hdfs.allowSnapshot(foo);
  hdfs.allowSnapshot(foo2);
  final Path bar = new Path(foo, "bar");
  final Path bar2 = new Path(foo2, "bar");
  DFSTestUtil.createFile(hdfs, bar, 100, (short) 2, 100024L);
  hdfs.createSnapshot(foo, "s1");
  // rename to another snapshottable directory and take snapshot
  assertTrue(hdfs.rename(bar, bar2));
  hdfs.createSnapshot(foo2, "s2");
  // delete the original renamed file to make sure blocks are not updated by
  // the original file
  assertTrue(hdfs.delete(bar2, true));

  // checkpoint
  NameNode nameNode = cluster.getNameNode();
  NameNodeAdapter.enterSafeMode(nameNode, false);
  NameNodeAdapter.saveNamespace(nameNode);
  NameNodeAdapter.leaveSafeMode(nameNode);
  // restart namenode to load snapshot files from fsimage
  cluster.restartNameNode(true);
  // file in first snapshot
  String barSnapshotPath = Snapshot.getSnapshotPath(foo.toString(), "s1/bar");
  DFSTestUtil.readFile(hdfs, new Path(barSnapshotPath));
  // file in second snapshot after rename+delete
  String bar2SnapshotPath = Snapshot.getSnapshotPath(foo2.toString(),
      "s2/bar");
  DFSTestUtil.readFile(hdfs, new Path(bar2SnapshotPath));
}
 

@Test(timeout=60000)
public void testRemoveOneVolume()
    throws ReconfigurationException, InterruptedException, TimeoutException,
    IOException {
  startDFSCluster(1, 1);
  final short replFactor = 1;
  Path testFile = new Path("/test");
  createFile(testFile, 10, replFactor);

  DataNode dn = cluster.getDataNodes().get(0);
  Collection<String> oldDirs = getDataDirs(dn);
  String newDirs = oldDirs.iterator().next();  // Keep the first volume.
  dn.reconfigurePropertyImpl(
      DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
  assertFileLocksReleased(
    new ArrayList<String>(oldDirs).subList(1, oldDirs.size()));
  dn.scheduleAllBlockReport(0);

  try {
    DFSTestUtil.readFile(cluster.getFileSystem(), testFile);
    fail("Expect to throw BlockMissingException.");
  } catch (BlockMissingException e) {
    GenericTestUtils.assertExceptionContains("Could not obtain block", e);
  }

  Path newFile = new Path("/newFile");
  createFile(newFile, 6);

  String bpid = cluster.getNamesystem().getBlockPoolId();
  List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
      cluster.getAllBlockReports(bpid);
  assertEquals((int)replFactor, blockReports.size());

  BlockListAsLongs blocksForVolume1 =
      blockReports.get(0).values().iterator().next();
  // The first volume has half of the testFile and full of newFile.
  assertEquals(10 / 2 + 6, blocksForVolume1.getNumberOfBlocks());
}
 

/**
 * This function ensures that writing causes TotalWritetime to increment
 * and reading causes totalReadTime to move.
 * @throws Exception
 */
@Test
public void testDataNodeTimeSpend() throws Exception {
  Configuration conf = new HdfsConfiguration();
  SimulatedFSDataset.setFactory(conf);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
  try {
    FileSystem fs = cluster.getFileSystem();
    List<DataNode> datanodes = cluster.getDataNodes();
    assertEquals(datanodes.size(), 1);
    DataNode datanode = datanodes.get(0);
    MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
    final long LONG_FILE_LEN = 1024 * 1024 * 10;

    long startWriteValue = getLongCounter("TotalWriteTime", rb);
    long startReadValue = getLongCounter("TotalReadTime", rb);

    for (int x =0; x < 50; x++) {
      DFSTestUtil.createFile(fs, new Path("/time.txt."+ x),
              LONG_FILE_LEN, (short) 1, Time.monotonicNow());
    }

    for (int x =0; x < 50; x++) {
      String s = DFSTestUtil.readFile(fs, new Path("/time.txt." + x));
    }

    MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name());
    long endWriteValue = getLongCounter("TotalWriteTime", rbNew);
    long endReadValue = getLongCounter("TotalReadTime", rbNew);

    assertTrue(endReadValue > startReadValue);
    assertTrue(endWriteValue > startWriteValue);
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test
public void testSendDataPacketMetrics() throws Exception {
  Configuration conf = new HdfsConfiguration();
  final int interval = 1;
  conf.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
  try {
    FileSystem fs = cluster.getFileSystem();
    // Create and read a 1 byte file
    Path tmpfile = new Path("/tmp.txt");
    DFSTestUtil.createFile(fs, tmpfile,
        (long)1, (short)1, 1L);
    DFSTestUtil.readFile(fs, tmpfile);
    List<DataNode> datanodes = cluster.getDataNodes();
    assertEquals(datanodes.size(), 1);
    DataNode datanode = datanodes.get(0);
    MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
    // Expect 2 packets, 1 for the 1 byte read, 1 for the empty packet
    // signaling the end of the block
    assertCounter("SendDataPacketTransferNanosNumOps", (long)2, rb);
    assertCounter("SendDataPacketBlockedOnNetworkNanosNumOps", (long)2, rb);
    // Wait for at least 1 rollover
    Thread.sleep((interval + 1) * 1000);
    // Check that the sendPacket percentiles rolled to non-zero values
    String sec = interval + "s";
    assertQuantileGauges("SendDataPacketBlockedOnNetworkNanos" + sec, rb);
    assertQuantileGauges("SendDataPacketTransferNanos" + sec, rb);
  } finally {
    if (cluster != null) {cluster.shutdown();}
  }
}
 

@Test(timeout = 30000)
public void testReadRenamedSnapshotFileWithCheckpoint() throws Exception {
  final Path foo = new Path("/foo");
  final Path foo2 = new Path("/foo2");
  hdfs.mkdirs(foo);
  hdfs.mkdirs(foo2);

  hdfs.allowSnapshot(foo);
  hdfs.allowSnapshot(foo2);
  final Path bar = new Path(foo, "bar");
  final Path bar2 = new Path(foo2, "bar");
  DFSTestUtil.createFile(hdfs, bar, 100, (short) 2, 100024L);
  hdfs.createSnapshot(foo, "s1");
  // rename to another snapshottable directory and take snapshot
  assertTrue(hdfs.rename(bar, bar2));
  hdfs.createSnapshot(foo2, "s2");
  // delete the original renamed file to make sure blocks are not updated by
  // the original file
  assertTrue(hdfs.delete(bar2, true));

  // checkpoint
  NameNode nameNode = cluster.getNameNode();
  NameNodeAdapter.enterSafeMode(nameNode, false);
  NameNodeAdapter.saveNamespace(nameNode);
  NameNodeAdapter.leaveSafeMode(nameNode);
  // restart namenode to load snapshot files from fsimage
  cluster.restartNameNode(true);
  // file in first snapshot
  String barSnapshotPath = Snapshot.getSnapshotPath(foo.toString(), "s1/bar");
  DFSTestUtil.readFile(hdfs, new Path(barSnapshotPath));
  // file in second snapshot after rename+delete
  String bar2SnapshotPath = Snapshot.getSnapshotPath(foo2.toString(),
      "s2/bar");
  DFSTestUtil.readFile(hdfs, new Path(bar2SnapshotPath));
}
 

/**
 * This function ensures that writing causes TotalWritetime to increment
 * and reading causes totalReadTime to move.
 * @throws Exception
 */
@Test
public void testDataNodeTimeSpend() throws Exception {
  Configuration conf = new HdfsConfiguration();
  SimulatedFSDataset.setFactory(conf);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
  try {
    FileSystem fs = cluster.getFileSystem();
    List<DataNode> datanodes = cluster.getDataNodes();
    assertEquals(datanodes.size(), 1);
    DataNode datanode = datanodes.get(0);
    MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
    final long LONG_FILE_LEN = 1024 * 1024 * 10;

    long startWriteValue = getLongCounter("TotalWriteTime", rb);
    long startReadValue = getLongCounter("TotalReadTime", rb);

    for (int x =0; x < 50; x++) {
      DFSTestUtil.createFile(fs, new Path("/time.txt."+ x),
              LONG_FILE_LEN, (short) 1, Time.monotonicNow());
    }

    for (int x =0; x < 50; x++) {
      String s = DFSTestUtil.readFile(fs, new Path("/time.txt." + x));
    }

    MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name());
    long endWriteValue = getLongCounter("TotalWriteTime", rbNew);
    long endReadValue = getLongCounter("TotalReadTime", rbNew);

    assertTrue(endReadValue > startReadValue);
    assertTrue(endWriteValue > startWriteValue);
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

@Test
public void testSendDataPacketMetrics() throws Exception {
  Configuration conf = new HdfsConfiguration();
  final int interval = 1;
  conf.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
  try {
    FileSystem fs = cluster.getFileSystem();
    // Create and read a 1 byte file
    Path tmpfile = new Path("/tmp.txt");
    DFSTestUtil.createFile(fs, tmpfile,
        (long)1, (short)1, 1L);
    DFSTestUtil.readFile(fs, tmpfile);
    List<DataNode> datanodes = cluster.getDataNodes();
    assertEquals(datanodes.size(), 1);
    DataNode datanode = datanodes.get(0);
    MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
    // Expect 2 packets, 1 for the 1 byte read, 1 for the empty packet
    // signaling the end of the block
    assertCounter("SendDataPacketTransferNanosNumOps", (long)2, rb);
    assertCounter("SendDataPacketBlockedOnNetworkNanosNumOps", (long)2, rb);
    // Wait for at least 1 rollover
    Thread.sleep((interval + 1) * 1000);
    // Check that the sendPacket percentiles rolled to non-zero values
    String sec = interval + "s";
    assertQuantileGauges("SendDataPacketBlockedOnNetworkNanos" + sec, rb);
    assertQuantileGauges("SendDataPacketTransferNanos" + sec, rb);
  } finally {
    if (cluster != null) {cluster.shutdown();}
  }
}
 

@Test (timeout=600000)
public void testDatanodeRollingUpgradeWithRollback() throws Exception {
  try {
    startCluster();

    // Create files in DFS.
    Path testFile1 = new Path("/" + GenericTestUtils.getMethodName() + ".01.dat");
    DFSTestUtil.createFile(fs, testFile1, FILE_SIZE, REPL_FACTOR, SEED);
    String fileContents1 = DFSTestUtil.readFile(fs, testFile1);

    startRollingUpgrade();

    File blockFile = getBlockForFile(testFile1, true);
    File trashFile = getTrashFileForBlock(blockFile, false);
    deleteAndEnsureInTrash(testFile1, blockFile, trashFile);

    // Now perform a rollback to restore DFS to the pre-rollback state.
    rollbackRollingUpgrade();

    // Ensure that block was restored from trash
    ensureTrashRestored(blockFile, trashFile);

    // Ensure that files exist and restored file contents are the same.
    assert(fs.exists(testFile1));
    String fileContents2 = DFSTestUtil.readFile(fs, testFile1);
    assertThat(fileContents1, is(fileContents2));
  } finally {
    shutdownCluster();
  }
}
 

@Test
public void testCorruptBlockRereplicatedAcrossRacks() throws Exception {
  Configuration conf = getConf();
  short REPLICATION_FACTOR = 2;
  int fileLen = 512;
  final Path filePath = new Path("/testFile");
  // Datanodes are spread across two racks
  String racks[] = {"/rack1", "/rack1", "/rack2", "/rack2"};
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
    .numDataNodes(racks.length).racks(racks).build();
  final FSNamesystem ns = cluster.getNameNode().getNamesystem();

  try {
    // Create a file with one block with a replication factor of 2
    final FileSystem fs = cluster.getFileSystem();
    
    DFSTestUtil.createFile(fs, filePath, fileLen, REPLICATION_FACTOR, 1L);
    final String fileContent = DFSTestUtil.readFile(fs, filePath);

    ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
    DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);

    // Corrupt a replica of the block
    int dnToCorrupt = DFSTestUtil.firstDnWithBlock(cluster, b);
    assertTrue(cluster.corruptReplica(dnToCorrupt, b));

    // Restart the datanode so blocks are re-scanned, and the corrupt
    // block is detected.
    cluster.restartDataNode(dnToCorrupt);

    // Wait for the namenode to notice the corrupt replica
    DFSTestUtil.waitCorruptReplicas(fs, ns, filePath, b, 1);

    // The rack policy is still respected
    DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);

    // Ensure all replicas are valid (the corrupt replica may not
    // have been cleaned up yet).
    for (int i = 0; i < racks.length; i++) {
      String blockContent = cluster.readBlockOnDataNode(i, b);
      if (blockContent != null && i != dnToCorrupt) {
        assertEquals("Corrupt replica", fileContent, blockContent);
      }
    }
  } finally {
    cluster.shutdown();
  }
}
 

/**
 * Verify that a file can be read both from NameNode and BackupNode.
 */
@Test
public void testCanReadData() throws IOException {
  Path file1 = new Path("/fileToRead.dat");
  Configuration conf = new HdfsConfiguration();
  MiniDFSCluster cluster = null;
  FileSystem fileSys = null;
  BackupNode backup = null;
  try {
    // Start NameNode and BackupNode
    cluster = new MiniDFSCluster.Builder(conf)
                                .numDataNodes(0).format(true).build();
    fileSys = cluster.getFileSystem();
    long txid = cluster.getNameNodeRpc().getTransactionID();
    backup = startBackupNode(conf, StartupOption.BACKUP, 1);
    waitCheckpointDone(cluster, txid);

    // Setup dual NameNode configuration for DataNodes
    String rpcAddrKeyPreffix =
        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + ".bnCluster";
    String nnAddr = cluster.getNameNode().getNameNodeAddressHostPortString();
        conf.get(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
    String bnAddr = backup.getNameNodeAddressHostPortString();
    conf.set(DFSConfigKeys.DFS_NAMESERVICES, "bnCluster");
    conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, "bnCluster");
    conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + ".bnCluster",
        "nnActive, nnBackup");
    conf.set(rpcAddrKeyPreffix + ".nnActive", nnAddr);
    conf.set(rpcAddrKeyPreffix + ".nnBackup", bnAddr);
    cluster.startDataNodes(conf, 3, true, StartupOption.REGULAR, null);

    DFSTestUtil.createFile(
        fileSys, file1, 8192, (short)3, 0);

    // Read the same file from file systems pointing to NN and BN
    FileSystem bnFS = FileSystem.get(
        new Path("hdfs://" + bnAddr).toUri(), conf);
    String nnData = DFSTestUtil.readFile(fileSys, file1);
    String bnData = DFSTestUtil.readFile(bnFS, file1);
    assertEquals("Data read from BackupNode and NameNode is not the same.",
        nnData, bnData);
  } catch(IOException e) {
    LOG.error("Error in TestBackupNode: ", e);
    assertTrue(e.getLocalizedMessage(), false);
  } finally {
    if(fileSys != null) fileSys.close();
    if(backup != null) backup.stop();
    if(cluster != null) cluster.shutdown();
  }
}
 

/**
 * Another regression test for HDFS-2742. This tests the following sequence:
 * - DN does a block report while file is open. This BR contains
 *   the block in RBW state.
 * - The block report is delayed in reaching the standby.
 * - The file is closed.
 * - The standby processes the OP_ADD and OP_CLOSE operations before
 *   the RBW block report arrives.
 * - The standby should not mark the block as corrupt.
 */
@Test
public void testRBWReportArrivesAfterEdits() throws Exception {
  final CountDownLatch brFinished = new CountDownLatch(1);
  DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG) {
    @Override
    protected Object passThrough(InvocationOnMock invocation)
        throws Throwable {
      try {
        return super.passThrough(invocation);
      } finally {
        // inform the test that our block report went through.
        brFinished.countDown();
      }
    }
  };

  FSDataOutputStream out = fs.create(TEST_FILE_PATH);
  try {
    AppendTestUtil.write(out, 0, 10);
    out.hflush();

    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeProtocolClientSideTranslatorPB spy =
      DataNodeTestUtils.spyOnBposToNN(dn, nn2);
    
    Mockito.doAnswer(delayer)
      .when(spy).blockReport(
        Mockito.<DatanodeRegistration>anyObject(),
        Mockito.anyString(),
        Mockito.<StorageBlockReport[]>anyObject(),
        Mockito.<BlockReportContext>anyObject());
    dn.scheduleAllBlockReport(0);
    delayer.waitForCall();
    
  } finally {
    IOUtils.closeStream(out);
  }

  cluster.transitionToStandby(0);
  cluster.transitionToActive(1);
  
  delayer.proceed();
  brFinished.await();
  
  // Verify that no replicas are marked corrupt, and that the
  // file is readable from the failed-over standby.
  BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager());
  BlockManagerTestUtil.updateState(nn2.getNamesystem().getBlockManager());
  assertEquals(0, nn1.getNamesystem().getCorruptReplicaBlocks());
  assertEquals(0, nn2.getNamesystem().getCorruptReplicaBlocks());
  
  DFSTestUtil.readFile(fs, TEST_FILE_PATH);
}
 

/**
 * Test for the case where one of the DNs in the pipeline is in the
 * process of doing a block report exactly when the block is closed.
 * In this case, the block report becomes delayed until after the
 * block is marked completed on the NN, and hence it reports an RBW
 * replica for a COMPLETE block. Such a report should not be marked
 * corrupt.
 * This is a regression test for HDFS-2791.
 */
@Test(timeout=300000)
public void testOneReplicaRbwReportArrivesAfterBlockCompleted() throws Exception {
  final CountDownLatch brFinished = new CountDownLatch(1);
  DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG) {
    @Override
    protected Object passThrough(InvocationOnMock invocation)
        throws Throwable {
      try {
        return super.passThrough(invocation);
      } finally {
        // inform the test that our block report went through.
        brFinished.countDown();
      }
    }
  };

  final String METHOD_NAME = GenericTestUtils.getMethodName();
  Path filePath = new Path("/" + METHOD_NAME + ".dat");

  // Start a second DN for this test -- we're checking
  // what happens when one of the DNs is slowed for some reason.
  REPL_FACTOR = 2;
  startDNandWait(null, false);

  NameNode nn = cluster.getNameNode();

  FSDataOutputStream out = fs.create(filePath, REPL_FACTOR);
  try {
    AppendTestUtil.write(out, 0, 10);
    out.hflush();

    // Set up a spy so that we can delay the block report coming
    // from this node.
    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeProtocolClientSideTranslatorPB spy =
      DataNodeTestUtils.spyOnBposToNN(dn, nn);

    Mockito.doAnswer(delayer)
      .when(spy).blockReport(
        Mockito.<DatanodeRegistration>anyObject(),
        Mockito.anyString(),
        Mockito.<StorageBlockReport[]>anyObject(),
        Mockito.<BlockReportContext>anyObject());

    // Force a block report to be generated. The block report will have
    // an RBW replica in it. Wait for the RPC to be sent, but block
    // it before it gets to the NN.
    dn.scheduleAllBlockReport(0);
    delayer.waitForCall();

  } finally {
    IOUtils.closeStream(out);
  }

  // Now that the stream is closed, the NN will have the block in COMPLETE
  // state.
  delayer.proceed();
  brFinished.await();

  // Verify that no replicas are marked corrupt, and that the
  // file is still readable.
  BlockManagerTestUtil.updateState(nn.getNamesystem().getBlockManager());
  assertEquals(0, nn.getNamesystem().getCorruptReplicaBlocks());
  DFSTestUtil.readFile(fs, filePath);

  // Ensure that the file is readable even from the DN that we futzed with.
  cluster.stopDataNode(1);
  DFSTestUtil.readFile(fs, filePath);
}
 

/**
 * Validates lazy persisted blocks are evicted from RAM_DISK based on LRU.
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testRamDiskEvictionIsLru()
  throws Exception {
  final int NUM_PATHS = 5;
  startUpCluster(true, NUM_PATHS + EVICTION_LOW_WATERMARK);
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  Path paths[] = new Path[NUM_PATHS * 2];

  for (int i = 0; i < paths.length; i++) {
    paths[i] = new Path("/" + METHOD_NAME + "." + i +".dat");
  }

  for (int i = 0; i < NUM_PATHS; i++) {
    makeTestFile(paths[i], BLOCK_SIZE, true);
  }

  // Sleep for a short time to allow the lazy writer thread to do its job.
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  for (int i = 0; i < NUM_PATHS; ++i) {
    ensureFileReplicasOnStorageType(paths[i], RAM_DISK);
  }

  // Open the files for read in a random order.
  ArrayList<Integer> indexes = new ArrayList<Integer>(NUM_PATHS);
  for (int i = 0; i < NUM_PATHS; ++i) {
    indexes.add(i);
  }
  Collections.shuffle(indexes);

  for (int i = 0; i < NUM_PATHS; ++i) {
    LOG.info("Touching file " + paths[indexes.get(i)]);
    DFSTestUtil.readFile(fs, paths[indexes.get(i)]);
  }

  // Create an equal number of new files ensuring that the previous
  // files are evicted in the same order they were read.
  for (int i = 0; i < NUM_PATHS; ++i) {
    makeTestFile(paths[i + NUM_PATHS], BLOCK_SIZE, true);
    triggerBlockReport();
    Thread.sleep(3000);
    ensureFileReplicasOnStorageType(paths[i + NUM_PATHS], RAM_DISK);
    ensureFileReplicasOnStorageType(paths[indexes.get(i)], DEFAULT);
    for (int j = i + 1; j < NUM_PATHS; ++j) {
      ensureFileReplicasOnStorageType(paths[indexes.get(j)], RAM_DISK);
    }
  }

  verifyRamDiskJMXMetric("RamDiskBlocksWrite", NUM_PATHS * 2);
  verifyRamDiskJMXMetric("RamDiskBlocksWriteFallback", 0);
  verifyRamDiskJMXMetric("RamDiskBytesWrite", BLOCK_SIZE * NUM_PATHS * 2);
  verifyRamDiskJMXMetric("RamDiskBlocksReadHits", NUM_PATHS);
  verifyRamDiskJMXMetric("RamDiskBlocksEvicted", NUM_PATHS);
  verifyRamDiskJMXMetric("RamDiskBlocksEvictedWithoutRead", 0);
  verifyRamDiskJMXMetric("RamDiskBlocksDeletedBeforeLazyPersisted", 0);
}
 

/**
 * Verify that a file can be read both from NameNode and BackupNode.
 */
@Test
public void testCanReadData() throws IOException {
  Path file1 = new Path("/fileToRead.dat");
  Configuration conf = new HdfsConfiguration();
  MiniDFSCluster cluster = null;
  FileSystem fileSys = null;
  BackupNode backup = null;
  try {
    // Start NameNode and BackupNode
    cluster = new MiniDFSCluster.Builder(conf)
                                .numDataNodes(0).format(true).build();
    fileSys = cluster.getFileSystem();
    long txid = cluster.getNameNodeRpc().getTransactionID();
    backup = startBackupNode(conf, StartupOption.BACKUP, 1);
    waitCheckpointDone(cluster, txid);

    // Setup dual NameNode configuration for DataNodes
    String rpcAddrKeyPreffix =
        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + ".bnCluster";
    String nnAddr = cluster.getNameNode().getNameNodeAddressHostPortString();
        conf.get(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
    String bnAddr = backup.getNameNodeAddressHostPortString();
    conf.set(DFSConfigKeys.DFS_NAMESERVICES, "bnCluster");
    conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, "bnCluster");
    conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + ".bnCluster",
        "nnActive, nnBackup");
    conf.set(rpcAddrKeyPreffix + ".nnActive", nnAddr);
    conf.set(rpcAddrKeyPreffix + ".nnBackup", bnAddr);
    cluster.startDataNodes(conf, 3, true, StartupOption.REGULAR, null);

    DFSTestUtil.createFile(
        fileSys, file1, 8192, (short)3, 0);

    // Read the same file from file systems pointing to NN and BN
    FileSystem bnFS = FileSystem.get(
        new Path("hdfs://" + bnAddr).toUri(), conf);
    String nnData = DFSTestUtil.readFile(fileSys, file1);
    String bnData = DFSTestUtil.readFile(bnFS, file1);
    assertEquals("Data read from BackupNode and NameNode is not the same.",
        nnData, bnData);
  } catch(IOException e) {
    LOG.error("Error in TestBackupNode: ", e);
    assertTrue(e.getLocalizedMessage(), false);
  } finally {
    if(fileSys != null) fileSys.close();
    if(backup != null) backup.stop();
    if(cluster != null) cluster.shutdown();
  }
}
 

/**
 * Validates lazy persisted blocks are evicted from RAM_DISK based on LRU.
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testRamDiskEvictionIsLru()
  throws Exception {
  final int NUM_PATHS = 5;
  startUpCluster(true, NUM_PATHS + EVICTION_LOW_WATERMARK);
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  Path paths[] = new Path[NUM_PATHS * 2];

  for (int i = 0; i < paths.length; i++) {
    paths[i] = new Path("/" + METHOD_NAME + "." + i +".dat");
  }

  for (int i = 0; i < NUM_PATHS; i++) {
    makeTestFile(paths[i], BLOCK_SIZE, true);
  }

  // Sleep for a short time to allow the lazy writer thread to do its job.
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  for (int i = 0; i < NUM_PATHS; ++i) {
    ensureFileReplicasOnStorageType(paths[i], RAM_DISK);
  }

  // Open the files for read in a random order.
  ArrayList<Integer> indexes = new ArrayList<Integer>(NUM_PATHS);
  for (int i = 0; i < NUM_PATHS; ++i) {
    indexes.add(i);
  }
  Collections.shuffle(indexes);

  for (int i = 0; i < NUM_PATHS; ++i) {
    LOG.info("Touching file " + paths[indexes.get(i)]);
    DFSTestUtil.readFile(fs, paths[indexes.get(i)]);
  }

  // Create an equal number of new files ensuring that the previous
  // files are evicted in the same order they were read.
  for (int i = 0; i < NUM_PATHS; ++i) {
    makeTestFile(paths[i + NUM_PATHS], BLOCK_SIZE, true);
    triggerBlockReport();
    Thread.sleep(3000);
    ensureFileReplicasOnStorageType(paths[i + NUM_PATHS], RAM_DISK);
    ensureFileReplicasOnStorageType(paths[indexes.get(i)], DEFAULT);
    for (int j = i + 1; j < NUM_PATHS; ++j) {
      ensureFileReplicasOnStorageType(paths[indexes.get(j)], RAM_DISK);
    }
  }

  verifyRamDiskJMXMetric("RamDiskBlocksWrite", NUM_PATHS * 2);
  verifyRamDiskJMXMetric("RamDiskBlocksWriteFallback", 0);
  verifyRamDiskJMXMetric("RamDiskBytesWrite", BLOCK_SIZE * NUM_PATHS * 2);
  verifyRamDiskJMXMetric("RamDiskBlocksReadHits", NUM_PATHS);
  verifyRamDiskJMXMetric("RamDiskBlocksEvicted", NUM_PATHS);
  verifyRamDiskJMXMetric("RamDiskBlocksEvictedWithoutRead", 0);
  verifyRamDiskJMXMetric("RamDiskBlocksDeletedBeforeLazyPersisted", 0);
}
 

/**
 * Another regression test for HDFS-2742. This tests the following sequence:
 * - DN does a block report while file is open. This BR contains
 *   the block in RBW state.
 * - The block report is delayed in reaching the standby.
 * - The file is closed.
 * - The standby processes the OP_ADD and OP_CLOSE operations before
 *   the RBW block report arrives.
 * - The standby should not mark the block as corrupt.
 */
@Test
public void testRBWReportArrivesAfterEdits() throws Exception {
  final CountDownLatch brFinished = new CountDownLatch(1);
  DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG) {
    @Override
    protected Object passThrough(InvocationOnMock invocation)
        throws Throwable {
      try {
        return super.passThrough(invocation);
      } finally {
        // inform the test that our block report went through.
        brFinished.countDown();
      }
    }
  };

  FSDataOutputStream out = fs.create(TEST_FILE_PATH);
  try {
    AppendTestUtil.write(out, 0, 10);
    out.hflush();

    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeProtocolClientSideTranslatorPB spy =
      DataNodeTestUtils.spyOnBposToNN(dn, nn2);
    
    Mockito.doAnswer(delayer)
      .when(spy).blockReport(
        Mockito.<DatanodeRegistration>anyObject(),
        Mockito.anyString(),
        Mockito.<StorageBlockReport[]>anyObject(),
        Mockito.<BlockReportContext>anyObject());
    dn.scheduleAllBlockReport(0);
    delayer.waitForCall();
    
  } finally {
    IOUtils.closeStream(out);
  }

  cluster.transitionToStandby(0);
  cluster.transitionToActive(1);
  
  delayer.proceed();
  brFinished.await();
  
  // Verify that no replicas are marked corrupt, and that the
  // file is readable from the failed-over standby.
  BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager());
  BlockManagerTestUtil.updateState(nn2.getNamesystem().getBlockManager());
  assertEquals(0, nn1.getNamesystem().getCorruptReplicaBlocks());
  assertEquals(0, nn2.getNamesystem().getCorruptReplicaBlocks());
  
  DFSTestUtil.readFile(fs, TEST_FILE_PATH);
}
 

/**
 * Test case that reduces replication of a file with a lot of blocks
 * and then fails over right after those blocks enter the DN invalidation
 * queues on the active. Ensures that fencing is correct and no replicas
 * are lost.
 */
@Test
public void testNNClearsCommandsOnFailoverWithReplChanges()
    throws Exception {
  // Make lots of blocks to increase chances of triggering a bug.
  DFSTestUtil.createFile(fs, TEST_FILE_PATH, 30*SMALL_BLOCK, (short)1, 1L);

  banner("rolling NN1's edit log, forcing catch-up");
  HATestUtil.waitForStandbyToCatchUp(nn1, nn2);
  
  // Get some new replicas reported so that NN2 now considers
  // them over-replicated and schedules some more deletions
  nn1.getRpcServer().setReplication(TEST_FILE, (short) 2);
  while (BlockManagerTestUtil.getComputedDatanodeWork(
      nn1.getNamesystem().getBlockManager()) > 0) {
    LOG.info("Getting more replication work computed");
  }
  BlockManager bm1 = nn1.getNamesystem().getBlockManager();
  while (bm1.getPendingReplicationBlocksCount() > 0) {
    BlockManagerTestUtil.updateState(bm1);
    cluster.triggerHeartbeats();
    Thread.sleep(1000);
  }
  
  banner("triggering BRs");
  cluster.triggerBlockReports();
  
  nn1.getRpcServer().setReplication(TEST_FILE, (short) 1);

  
  banner("computing invalidation on nn1");

  BlockManagerTestUtil.computeInvalidationWork(
      nn1.getNamesystem().getBlockManager());
  doMetasave(nn1);

  banner("computing invalidation on nn2");
  BlockManagerTestUtil.computeInvalidationWork(
      nn2.getNamesystem().getBlockManager());
  doMetasave(nn2);

  // Dump some info for debugging purposes.
  banner("Metadata immediately before failover");
  doMetasave(nn2);


  // Transition nn2 to active even though nn1 still thinks it's active
  banner("Failing to NN2 but let NN1 continue to think it's active");
  NameNodeAdapter.abortEditLogs(nn1);
  NameNodeAdapter.enterSafeMode(nn1, false);

  
  BlockManagerTestUtil.computeInvalidationWork(
      nn2.getNamesystem().getBlockManager());
  cluster.transitionToActive(1);

  // Check that the standby picked up the replication change.
  assertEquals(1,
      nn2.getRpcServer().getFileInfo(TEST_FILE).getReplication());

  // Dump some info for debugging purposes.
  banner("Metadata immediately after failover");
  doMetasave(nn2);
  
  banner("Triggering heartbeats and block reports so that fencing is completed");
  cluster.triggerHeartbeats();
  cluster.triggerBlockReports();
  
  banner("Metadata after nodes have all block-reported");
  doMetasave(nn2);
  
  // Force a rescan of postponedMisreplicatedBlocks.
  BlockManager nn2BM = nn2.getNamesystem().getBlockManager();
  BlockManagerTestUtil.checkHeartbeat(nn2BM);
  BlockManagerTestUtil.rescanPostponedMisreplicatedBlocks(nn2BM);

  // The block should no longer be postponed.
  assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks());
  
  // Wait for NN2 to enact its deletions (replication monitor has to run, etc)
  BlockManagerTestUtil.computeInvalidationWork(
      nn2.getNamesystem().getBlockManager());

  HATestUtil.waitForNNToIssueDeletions(nn2);
  cluster.triggerHeartbeats();
  HATestUtil.waitForDNDeletions(cluster);
  cluster.triggerDeletionReports();
  assertEquals(0, nn2.getNamesystem().getUnderReplicatedBlocks());
  assertEquals(0, nn2.getNamesystem().getPendingReplicationBlocks());
  
  banner("Making sure the file is still readable");
  FileSystem fs2 = cluster.getFileSystem(1);
  DFSTestUtil.readFile(fs2, TEST_FILE_PATH);
}
 

@Test
public void testDnFencing() throws Exception {
  // Create a file with replication level 3.
  DFSTestUtil.createFile(fs, TEST_FILE_PATH, 30*SMALL_BLOCK, (short)3, 1L);
  ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, TEST_FILE_PATH);
  
  // Drop its replication count to 1, so it becomes over-replicated.
  // Then compute the invalidation of the extra blocks and trigger
  // heartbeats so the invalidations are flushed to the DNs.
  nn1.getRpcServer().setReplication(TEST_FILE, (short) 1);
  BlockManagerTestUtil.computeInvalidationWork(
      nn1.getNamesystem().getBlockManager());
  cluster.triggerHeartbeats();
  
  // Transition nn2 to active even though nn1 still thinks it's active.
  banner("Failing to NN2 but let NN1 continue to think it's active");
  NameNodeAdapter.abortEditLogs(nn1);
  NameNodeAdapter.enterSafeMode(nn1, false);
  cluster.transitionToActive(1);
  
  // Check that the standby picked up the replication change.
  assertEquals(1,
      nn2.getRpcServer().getFileInfo(TEST_FILE).getReplication());

  // Dump some info for debugging purposes.
  banner("NN2 Metadata immediately after failover");
  doMetasave(nn2);
  
  banner("Triggering heartbeats and block reports so that fencing is completed");
  cluster.triggerHeartbeats();
  cluster.triggerBlockReports();
  
  banner("Metadata after nodes have all block-reported");
  doMetasave(nn2);

  // Force a rescan of postponedMisreplicatedBlocks.
  BlockManager nn2BM = nn2.getNamesystem().getBlockManager();
  BlockManagerTestUtil.checkHeartbeat(nn2BM);
  BlockManagerTestUtil.rescanPostponedMisreplicatedBlocks(nn2BM);

  // The blocks should no longer be postponed.
  assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks());
  
  // Wait for NN2 to enact its deletions (replication monitor has to run, etc)
  BlockManagerTestUtil.computeInvalidationWork(
      nn2.getNamesystem().getBlockManager());
  cluster.triggerHeartbeats();
  HATestUtil.waitForDNDeletions(cluster);
  cluster.triggerDeletionReports();
  assertEquals(0, nn2.getNamesystem().getUnderReplicatedBlocks());
  assertEquals(0, nn2.getNamesystem().getPendingReplicationBlocks());
  
  banner("Making sure the file is still readable");
  FileSystem fs2 = cluster.getFileSystem(1);
  DFSTestUtil.readFile(fs2, TEST_FILE_PATH);

  banner("Waiting for the actual block files to get deleted from DNs.");
  waitForTrueReplication(cluster, block, 1);
}
 

/**
 * Test case which restarts the standby node in such a way that,
 * when it exits safemode, it will want to invalidate a bunch
 * of over-replicated block replicas. Ensures that if we failover
 * at this point it won't lose data.
 */
@Test
public void testNNClearsCommandsOnFailoverAfterStartup()
    throws Exception {
  // Make lots of blocks to increase chances of triggering a bug.
  DFSTestUtil.createFile(fs, TEST_FILE_PATH, 30*SMALL_BLOCK, (short)3, 1L);

  banner("Shutting down NN2");
  cluster.shutdownNameNode(1);

  banner("Setting replication to 1, rolling edit log.");
  nn1.getRpcServer().setReplication(TEST_FILE, (short) 1);
  nn1.getRpcServer().rollEditLog();
  
  // Start NN2 again. When it starts up, it will see all of the
  // blocks as over-replicated, since it has the metadata for
  // replication=1, but the DNs haven't yet processed the deletions.
  banner("Starting NN2 again.");
  cluster.restartNameNode(1);
  nn2 = cluster.getNameNode(1);
  
  banner("triggering BRs");
  cluster.triggerBlockReports();

  // We expect that both NN1 and NN2 will have some number of
  // deletions queued up for the DNs.
  banner("computing invalidation on nn1");
  BlockManagerTestUtil.computeInvalidationWork(
      nn1.getNamesystem().getBlockManager());

  banner("computing invalidation on nn2");
  BlockManagerTestUtil.computeInvalidationWork(
      nn2.getNamesystem().getBlockManager());
  
  // Dump some info for debugging purposes.
  banner("Metadata immediately before failover");
  doMetasave(nn2);


  // Transition nn2 to active even though nn1 still thinks it's active
  banner("Failing to NN2 but let NN1 continue to think it's active");
  NameNodeAdapter.abortEditLogs(nn1);
  NameNodeAdapter.enterSafeMode(nn1, false);

  cluster.transitionToActive(1);

  // Check that the standby picked up the replication change.
  assertEquals(1,
      nn2.getRpcServer().getFileInfo(TEST_FILE).getReplication());

  // Dump some info for debugging purposes.
  banner("Metadata immediately after failover");
  doMetasave(nn2);
  
  banner("Triggering heartbeats and block reports so that fencing is completed");
  cluster.triggerHeartbeats();
  cluster.triggerBlockReports();
  
  banner("Metadata after nodes have all block-reported");
  doMetasave(nn2);

  // Force a rescan of postponedMisreplicatedBlocks.
  BlockManager nn2BM = nn2.getNamesystem().getBlockManager();
  BlockManagerTestUtil.checkHeartbeat(nn2BM);
  BlockManagerTestUtil.rescanPostponedMisreplicatedBlocks(nn2BM);

  // The block should no longer be postponed.
  assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks());
  
  // Wait for NN2 to enact its deletions (replication monitor has to run, etc)
  BlockManagerTestUtil.computeInvalidationWork(
      nn2.getNamesystem().getBlockManager());

  HATestUtil.waitForNNToIssueDeletions(nn2);
  cluster.triggerHeartbeats();
  HATestUtil.waitForDNDeletions(cluster);
  cluster.triggerDeletionReports();
  assertEquals(0, nn2.getNamesystem().getUnderReplicatedBlocks());
  assertEquals(0, nn2.getNamesystem().getPendingReplicationBlocks());
  
  banner("Making sure the file is still readable");
  FileSystem fs2 = cluster.getFileSystem(1);
  DFSTestUtil.readFile(fs2, TEST_FILE_PATH);
}
 

@Test
public void testCorruptBlockRereplicatedAcrossRacks() throws Exception {
  Configuration conf = getConf();
  short REPLICATION_FACTOR = 2;
  int fileLen = 512;
  final Path filePath = new Path("/testFile");
  // Datanodes are spread across two racks
  String racks[] = {"/rack1", "/rack1", "/rack2", "/rack2"};
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
    .numDataNodes(racks.length).racks(racks).build();
  final FSNamesystem ns = cluster.getNameNode().getNamesystem();

  try {
    // Create a file with one block with a replication factor of 2
    final FileSystem fs = cluster.getFileSystem();
    
    DFSTestUtil.createFile(fs, filePath, fileLen, REPLICATION_FACTOR, 1L);
    final String fileContent = DFSTestUtil.readFile(fs, filePath);

    ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
    DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);

    // Corrupt a replica of the block
    int dnToCorrupt = DFSTestUtil.firstDnWithBlock(cluster, b);
    assertTrue(cluster.corruptReplica(dnToCorrupt, b));

    // Restart the datanode so blocks are re-scanned, and the corrupt
    // block is detected.
    cluster.restartDataNode(dnToCorrupt);

    // Wait for the namenode to notice the corrupt replica
    DFSTestUtil.waitCorruptReplicas(fs, ns, filePath, b, 1);

    // The rack policy is still respected
    DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);

    // Ensure all replicas are valid (the corrupt replica may not
    // have been cleaned up yet).
    for (int i = 0; i < racks.length; i++) {
      String blockContent = cluster.readBlockOnDataNode(i, b);
      if (blockContent != null && i != dnToCorrupt) {
        assertEquals("Corrupt replica", fileContent, blockContent);
      }
    }
  } finally {
    cluster.shutdown();
  }
}