Java Code Examples for org.apache.hadoop.test.GenericTestUtils#DelayAnswer

/**
 * Another regression test for HDFS-2742. This tests the following sequence:
 * - DN does a block report while file is open. This BR contains
 *   the block in RBW state.
 * - The block report is delayed in reaching the standby.
 * - The file is closed.
 * - The standby processes the OP_ADD and OP_CLOSE operations before
 *   the RBW block report arrives.
 * - The standby should not mark the block as corrupt.
 */
@Test
public void testRBWReportArrivesAfterEdits() throws Exception {
  final CountDownLatch brFinished = new CountDownLatch(1);
  DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG) {
    @Override
    protected Object passThrough(InvocationOnMock invocation)
        throws Throwable {
      try {
        return super.passThrough(invocation);
      } finally {
        // inform the test that our block report went through.
        brFinished.countDown();
      }
    }
  };

  FSDataOutputStream out = fs.create(TEST_FILE_PATH);
  try {
    AppendTestUtil.write(out, 0, 10);
    out.hflush();

    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeProtocolClientSideTranslatorPB spy =
      DataNodeTestUtils.spyOnBposToNN(dn, nn2);
    
    Mockito.doAnswer(delayer)
      .when(spy).blockReport(
        Mockito.<DatanodeRegistration>anyObject(),
        Mockito.anyString(),
        Mockito.<StorageBlockReport[]>anyObject(),
        Mockito.<BlockReportContext>anyObject());
    dn.scheduleAllBlockReport(0);
    delayer.waitForCall();
    
  } finally {
    IOUtils.closeStream(out);
  }

  cluster.transitionToStandby(0);
  cluster.transitionToActive(1);
  
  delayer.proceed();
  brFinished.await();
  
  // Verify that no replicas are marked corrupt, and that the
  // file is readable from the failed-over standby.
  BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager());
  BlockManagerTestUtil.updateState(nn2.getNamesystem().getBlockManager());
  assertEquals(0, nn1.getNamesystem().getCorruptReplicaBlocks());
  assertEquals(0, nn2.getNamesystem().getCorruptReplicaBlocks());
  
  DFSTestUtil.readFile(fs, TEST_FILE_PATH);
}
 

@Test(timeout=20000)
public void testCancelSaveNamespace() throws Exception {
  Configuration conf = getConf();
  NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
  DFSTestUtil.formatNameNode(conf);
  FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);

  // Replace the FSImage with a spy
  final FSImage image = fsn.getFSImage();
  NNStorage storage = image.getStorage();
  storage.close(); // unlock any directories that FSNamesystem's initialization may have locked
  storage.setStorageDirectories(
      FSNamesystem.getNamespaceDirs(conf), 
      FSNamesystem.getNamespaceEditsDirs(conf));

  FSNamesystem spyFsn = spy(fsn);
  final FSNamesystem finalFsn = spyFsn;
  DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG);
  BlockIdManager bid = spy(spyFsn.getBlockIdManager());
  Whitebox.setInternalState(finalFsn, "blockIdManager", bid);
  doAnswer(delayer).when(bid).getGenerationStampV2();

  ExecutorService pool = Executors.newFixedThreadPool(2);
  
  try {
    doAnEdit(fsn, 1);
    final Canceler canceler = new Canceler();
    
    // Save namespace
    fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    try {
      Future<Void> saverFuture = pool.submit(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
          image.saveNamespace(finalFsn, NameNodeFile.IMAGE, canceler);
          return null;
        }
      });

      // Wait until saveNamespace calls getGenerationStamp
      delayer.waitForCall();
      // then cancel the saveNamespace
      Future<Void> cancelFuture = pool.submit(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
          canceler.cancel("cancelled");
          return null;
        }
      });
      // give the cancel call time to run
      Thread.sleep(500);
      
      // allow saveNamespace to proceed - it should check the cancel flag after
      // this point and throw an exception
      delayer.proceed();

      cancelFuture.get();
      saverFuture.get();
      fail("saveNamespace did not fail even though cancelled!");
    } catch (Throwable t) {
      GenericTestUtils.assertExceptionContains(
          "SaveNamespaceCancelledException", t);
    }
    LOG.info("Successfully cancelled a saveNamespace");


    // Check that we have only the original image and not any
    // cruft left over from half-finished images
    FSImageTestUtil.logStorageContents(LOG, storage);
    for (StorageDirectory sd : storage.dirIterable(null)) {
      File curDir = sd.getCurrentDir();
      GenericTestUtils.assertGlobEquals(curDir, "fsimage_.*",
          NNStorage.getImageFileName(0),
          NNStorage.getImageFileName(0) + MD5FileUtils.MD5_SUFFIX);
    }      
  } finally {
    fsn.close();
  }
}
 

/**
 * Test for the case where one of the DNs in the pipeline is in the
 * process of doing a block report exactly when the block is closed.
 * In this case, the block report becomes delayed until after the
 * block is marked completed on the NN, and hence it reports an RBW
 * replica for a COMPLETE block. Such a report should not be marked
 * corrupt.
 * This is a regression test for HDFS-2791.
 */
@Test(timeout=300000)
public void testOneReplicaRbwReportArrivesAfterBlockCompleted() throws Exception {
  final CountDownLatch brFinished = new CountDownLatch(1);
  DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG) {
    @Override
    protected Object passThrough(InvocationOnMock invocation)
        throws Throwable {
      try {
        return super.passThrough(invocation);
      } finally {
        // inform the test that our block report went through.
        brFinished.countDown();
      }
    }
  };

  final String METHOD_NAME = GenericTestUtils.getMethodName();
  Path filePath = new Path("/" + METHOD_NAME + ".dat");

  // Start a second DN for this test -- we're checking
  // what happens when one of the DNs is slowed for some reason.
  REPL_FACTOR = 2;
  startDNandWait(null, false);

  NameNode nn = cluster.getNameNode();

  FSDataOutputStream out = fs.create(filePath, REPL_FACTOR);
  try {
    AppendTestUtil.write(out, 0, 10);
    out.hflush();

    // Set up a spy so that we can delay the block report coming
    // from this node.
    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeProtocolClientSideTranslatorPB spy =
      DataNodeTestUtils.spyOnBposToNN(dn, nn);

    Mockito.doAnswer(delayer)
      .when(spy).blockReport(
        Mockito.<DatanodeRegistration>anyObject(),
        Mockito.anyString(),
        Mockito.<StorageBlockReport[]>anyObject(),
        Mockito.<BlockReportContext>anyObject());

    // Force a block report to be generated. The block report will have
    // an RBW replica in it. Wait for the RPC to be sent, but block
    // it before it gets to the NN.
    dn.scheduleAllBlockReport(0);
    delayer.waitForCall();

  } finally {
    IOUtils.closeStream(out);
  }

  // Now that the stream is closed, the NN will have the block in COMPLETE
  // state.
  delayer.proceed();
  brFinished.await();

  // Verify that no replicas are marked corrupt, and that the
  // file is still readable.
  BlockManagerTestUtil.updateState(nn.getNamesystem().getBlockManager());
  assertEquals(0, nn.getNamesystem().getCorruptReplicaBlocks());
  DFSTestUtil.readFile(fs, filePath);

  // Ensure that the file is readable even from the DN that we futzed with.
  cluster.stopDataNode(1);
  DFSTestUtil.readFile(fs, filePath);
}
 

/**
 * Another regression test for HDFS-2742. This tests the following sequence:
 * - DN does a block report while file is open. This BR contains
 *   the block in RBW state.
 * - The block report is delayed in reaching the standby.
 * - The file is closed.
 * - The standby processes the OP_ADD and OP_CLOSE operations before
 *   the RBW block report arrives.
 * - The standby should not mark the block as corrupt.
 */
@Test
public void testRBWReportArrivesAfterEdits() throws Exception {
  final CountDownLatch brFinished = new CountDownLatch(1);
  DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG) {
    @Override
    protected Object passThrough(InvocationOnMock invocation)
        throws Throwable {
      try {
        return super.passThrough(invocation);
      } finally {
        // inform the test that our block report went through.
        brFinished.countDown();
      }
    }
  };

  FSDataOutputStream out = fs.create(TEST_FILE_PATH);
  try {
    AppendTestUtil.write(out, 0, 10);
    out.hflush();

    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeProtocolClientSideTranslatorPB spy =
      DataNodeTestUtils.spyOnBposToNN(dn, nn2);
    
    Mockito.doAnswer(delayer)
      .when(spy).blockReport(
        Mockito.<DatanodeRegistration>anyObject(),
        Mockito.anyString(),
        Mockito.<StorageBlockReport[]>anyObject(),
        Mockito.<BlockReportContext>anyObject());
    dn.scheduleAllBlockReport(0);
    delayer.waitForCall();
    
  } finally {
    IOUtils.closeStream(out);
  }

  cluster.transitionToStandby(0);
  cluster.transitionToActive(1);
  
  delayer.proceed();
  brFinished.await();
  
  // Verify that no replicas are marked corrupt, and that the
  // file is readable from the failed-over standby.
  BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager());
  BlockManagerTestUtil.updateState(nn2.getNamesystem().getBlockManager());
  assertEquals(0, nn1.getNamesystem().getCorruptReplicaBlocks());
  assertEquals(0, nn2.getNamesystem().getCorruptReplicaBlocks());
  
  DFSTestUtil.readFile(fs, TEST_FILE_PATH);
}
 

@Test(timeout=20000)
public void testCancelSaveNamespace() throws Exception {
  Configuration conf = getConf();
  NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
  DFSTestUtil.formatNameNode(conf);
  FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);

  // Replace the FSImage with a spy
  final FSImage image = fsn.getFSImage();
  NNStorage storage = image.getStorage();
  storage.close(); // unlock any directories that FSNamesystem's initialization may have locked
  storage.setStorageDirectories(
      FSNamesystem.getNamespaceDirs(conf), 
      FSNamesystem.getNamespaceEditsDirs(conf));

  FSNamesystem spyFsn = spy(fsn);
  final FSNamesystem finalFsn = spyFsn;
  DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG);
  BlockIdManager bid = spy(spyFsn.getBlockIdManager());
  Whitebox.setInternalState(finalFsn, "blockIdManager", bid);
  doAnswer(delayer).when(bid).getGenerationStampV2();

  ExecutorService pool = Executors.newFixedThreadPool(2);
  
  try {
    doAnEdit(fsn, 1);
    final Canceler canceler = new Canceler();
    
    // Save namespace
    fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    try {
      Future<Void> saverFuture = pool.submit(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
          image.saveNamespace(finalFsn, NameNodeFile.IMAGE, canceler);
          return null;
        }
      });

      // Wait until saveNamespace calls getGenerationStamp
      delayer.waitForCall();
      // then cancel the saveNamespace
      Future<Void> cancelFuture = pool.submit(new Callable<Void>() {
        @Override
        public Void call() throws Exception {
          canceler.cancel("cancelled");
          return null;
        }
      });
      // give the cancel call time to run
      Thread.sleep(500);
      
      // allow saveNamespace to proceed - it should check the cancel flag after
      // this point and throw an exception
      delayer.proceed();

      cancelFuture.get();
      saverFuture.get();
      fail("saveNamespace did not fail even though cancelled!");
    } catch (Throwable t) {
      GenericTestUtils.assertExceptionContains(
          "SaveNamespaceCancelledException", t);
    }
    LOG.info("Successfully cancelled a saveNamespace");


    // Check that we have only the original image and not any
    // cruft left over from half-finished images
    FSImageTestUtil.logStorageContents(LOG, storage);
    for (StorageDirectory sd : storage.dirIterable(null)) {
      File curDir = sd.getCurrentDir();
      GenericTestUtils.assertGlobEquals(curDir, "fsimage_.*",
          NNStorage.getImageFileName(0),
          NNStorage.getImageFileName(0) + MD5FileUtils.MD5_SUFFIX);
    }      
  } finally {
    fsn.close();
  }
}
 

/**
 * Test for the case where one of the DNs in the pipeline is in the
 * process of doing a block report exactly when the block is closed.
 * In this case, the block report becomes delayed until after the
 * block is marked completed on the NN, and hence it reports an RBW
 * replica for a COMPLETE block. Such a report should not be marked
 * corrupt.
 * This is a regression test for HDFS-2791.
 */
@Test(timeout=300000)
public void testOneReplicaRbwReportArrivesAfterBlockCompleted() throws Exception {
  final CountDownLatch brFinished = new CountDownLatch(1);
  DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG) {
    @Override
    protected Object passThrough(InvocationOnMock invocation)
        throws Throwable {
      try {
        return super.passThrough(invocation);
      } finally {
        // inform the test that our block report went through.
        brFinished.countDown();
      }
    }
  };

  final String METHOD_NAME = GenericTestUtils.getMethodName();
  Path filePath = new Path("/" + METHOD_NAME + ".dat");

  // Start a second DN for this test -- we're checking
  // what happens when one of the DNs is slowed for some reason.
  REPL_FACTOR = 2;
  startDNandWait(null, false);

  NameNode nn = cluster.getNameNode();

  FSDataOutputStream out = fs.create(filePath, REPL_FACTOR);
  try {
    AppendTestUtil.write(out, 0, 10);
    out.hflush();

    // Set up a spy so that we can delay the block report coming
    // from this node.
    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeProtocolClientSideTranslatorPB spy =
      DataNodeTestUtils.spyOnBposToNN(dn, nn);

    Mockito.doAnswer(delayer)
      .when(spy).blockReport(
        Mockito.<DatanodeRegistration>anyObject(),
        Mockito.anyString(),
        Mockito.<StorageBlockReport[]>anyObject(),
        Mockito.<BlockReportContext>anyObject());

    // Force a block report to be generated. The block report will have
    // an RBW replica in it. Wait for the RPC to be sent, but block
    // it before it gets to the NN.
    dn.scheduleAllBlockReport(0);
    delayer.waitForCall();

  } finally {
    IOUtils.closeStream(out);
  }

  // Now that the stream is closed, the NN will have the block in COMPLETE
  // state.
  delayer.proceed();
  brFinished.await();

  // Verify that no replicas are marked corrupt, and that the
  // file is still readable.
  BlockManagerTestUtil.updateState(nn.getNamesystem().getBlockManager());
  assertEquals(0, nn.getNamesystem().getCorruptReplicaBlocks());
  DFSTestUtil.readFile(fs, filePath);

  // Ensure that the file is readable even from the DN that we futzed with.
  cluster.stopDataNode(1);
  DFSTestUtil.readFile(fs, filePath);
}