org.apache.hadoop.conf.ReconfigurationException Java Examples

/**
 * Reconfigure a DataNode by setting a new list of volumes.
 *
 * @param dn DataNode to reconfigure
 * @param newVols new volumes to configure
 * @throws Exception if there is any failure
 */
private static void reconfigureDataNode(DataNode dn, File... newVols)
    throws Exception {
  StringBuilder dnNewDataDirs = new StringBuilder();
  for (File newVol: newVols) {
    if (dnNewDataDirs.length() > 0) {
      dnNewDataDirs.append(',');
    }
    dnNewDataDirs.append(newVol.getAbsolutePath());
  }
  try {
    dn.reconfigurePropertyImpl(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
        dnNewDataDirs.toString());
  } catch (ReconfigurationException e) {
    // This can be thrown if reconfiguration tries to use a failed volume.
    // We need to swallow the exception, because some of our tests want to
    // cover this case.
    LOG.warn("Could not reconfigure DataNode.", e);
  }
}
 

/**
 * {@inheritDoc}
 */
@Override
public void reconfigurePropertyImpl(String property, String newVal) 
  throws ReconfigurationException {
  if (property.equals("dfs.data.dir")) {
    try {
      LOG.info("Reconfigure " + property + " to " + newVal);
      this.refreshVolumes(newVal);
    } catch (Exception e) {
      throw new ReconfigurationException(property, 
      newVal, getConf().get(property), e);
    }
  } else {
    throw new ReconfigurationException(property, newVal,
                                      getConf().get(property));
  }
}
 

@Override
public final String reconfigureProperty(String property, String newVal)
  throws ReconfigurationException {
  if (isPropertyReconfigurable(property)) {
    String oldVal;
    synchronized(getConf()) {
      oldVal = getConf().get(property);
      reconfigurePropertyImpl(property, newVal);
      if (newVal != null) {
        getConf().set(property, newVal);
      } else {
        getConf().unset(property);
      }
    }
    return oldVal;
  } else {
    throw new ReconfigurationException(property, newVal,
                                           getConf().get(property));
  }
}
 

@Override
protected void reconfigurePropertyImpl(String property, String newVal)
  throws ReconfigurationException {
  try {
    if (property.equals(MAX_TRACKER_BLACKLISTS_PROPERTY)) {
      LOG.info("changing maxTrackerBlacklists to " + newVal +
          " from " + initialJobRefreshTimeoutMs);
      maxTrackerBlacklists = Integer.parseInt(newVal);
    } else if (property.equals(MAX_UNIQUE_COUNTER_NAMES)) {
      LOG.info("changing maxUniqueCounterNames to " + newVal +
          " from " + initialJobRefreshTimeoutMs);
      maxUniqueCounterNames = Integer.parseInt(newVal);
    } else if (property.equals(INITIAL_JOB_REFRESH_TIMEOUT_MS_PROPERTY)) {
      LOG.info("changing initialJobRefreshTimeoutMs to " + newVal +
          " from " + initialJobRefreshTimeoutMs);
      initialJobRefreshTimeoutMs = Long.parseLong(newVal);
    }
  } catch (NumberFormatException e) {
    LOG.warn("reconfigurePropertyImpl: Invalid property " + property +
        " or newVal " + newVal);
  }
}
 

@Test(timeout=60000)
public void testReplicatingAfterRemoveVolume()
    throws InterruptedException, TimeoutException, IOException,
    ReconfigurationException {
  startDFSCluster(1, 2);

  final FileSystem fs = cluster.getFileSystem();
  final short replFactor = 2;
  Path testFile = new Path("/test");
  createFile(testFile, 4, replFactor);

  DataNode dn = cluster.getDataNodes().get(0);
  Collection<String> oldDirs = getDataDirs(dn);
  String newDirs = oldDirs.iterator().next();  // Keep the first volume.
  dn.reconfigurePropertyImpl(
      DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
  assertFileLocksReleased(
      new ArrayList<String>(oldDirs).subList(1, oldDirs.size()));

  triggerDeleteReport(dn);

  waitReplication(fs, testFile, 1, 1);
  DFSTestUtil.waitReplication(fs, testFile, replFactor);
}
 

/**
 * Reconfigure a DataNode by setting a new list of volumes.
 *
 * @param dn DataNode to reconfigure
 * @param newVols new volumes to configure
 * @throws Exception if there is any failure
 */
private static void reconfigureDataNode(DataNode dn, File... newVols)
    throws Exception {
  StringBuilder dnNewDataDirs = new StringBuilder();
  for (File newVol: newVols) {
    if (dnNewDataDirs.length() > 0) {
      dnNewDataDirs.append(',');
    }
    dnNewDataDirs.append(newVol.getAbsolutePath());
  }
  try {
    dn.reconfigurePropertyImpl(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
        dnNewDataDirs.toString());
  } catch (ReconfigurationException e) {
    // This can be thrown if reconfiguration tries to use a failed volume.
    // We need to swallow the exception, because some of our tests want to
    // cover this case.
    LOG.warn("Could not reconfigure DataNode.", e);
  }
}
 

@Override
public void reconfigurePropertyImpl(String property, String newVal)
    throws ReconfigurationException {
  if (property.equals(DFS_DATANODE_DATA_DIR_KEY)) {
    try {
      LOG.info("Reconfiguring " + property + " to " + newVal);
      this.refreshVolumes(newVal);
    } catch (IOException e) {
      throw new ReconfigurationException(property, newVal,
          getConf().get(property), e);
    }
  } else {
    throw new ReconfigurationException(
        property, newVal, getConf().get(property));
  }
}
 

@Override
public void reconfigurePropertyImpl(String property, String newVal)
    throws ReconfigurationException {
  if (property.equals(DFS_DATANODE_DATA_DIR_KEY)) {
    try {
      LOG.info("Reconfiguring " + property + " to " + newVal);
      this.refreshVolumes(newVal);
    } catch (IOException e) {
      throw new ReconfigurationException(property, newVal,
          getConf().get(property), e);
    }
  } else {
    throw new ReconfigurationException(
        property, newVal, getConf().get(property));
  }
}
 

@Test(timeout=60000)
public void testReplicatingAfterRemoveVolume()
    throws InterruptedException, TimeoutException, IOException,
    ReconfigurationException {
  startDFSCluster(1, 2);

  final FileSystem fs = cluster.getFileSystem();
  final short replFactor = 2;
  Path testFile = new Path("/test");
  createFile(testFile, 4, replFactor);

  DataNode dn = cluster.getDataNodes().get(0);
  Collection<String> oldDirs = getDataDirs(dn);
  String newDirs = oldDirs.iterator().next();  // Keep the first volume.
  dn.reconfigurePropertyImpl(
      DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
  assertFileLocksReleased(
      new ArrayList<String>(oldDirs).subList(1, oldDirs.size()));

  triggerDeleteReport(dn);

  waitReplication(fs, testFile, 1, 1);
  DFSTestUtil.waitReplication(fs, testFile, replFactor);
}
 

@Test(timeout=60000)
public void testAddVolumesToFederationNN()
    throws IOException, TimeoutException, InterruptedException,
    ReconfigurationException {
  // Starts a Cluster with 2 NameNode and 3 DataNodes. Each DataNode has 2
  // volumes.
  final int numNameNodes = 2;
  final int numDataNodes = 1;
  startDFSCluster(numNameNodes, numDataNodes);
  Path testFile = new Path("/test");
  // Create a file on the first namespace with 4 blocks.
  createFile(0, testFile, 4);
  // Create a file on the second namespace with 4 blocks.
  createFile(1, testFile, 4);

  // Add 2 volumes to the first DataNode.
  final int numNewVolumes = 2;
  addVolumes(numNewVolumes);

  // Append to the file on the first namespace.
  DFSTestUtil.appendFile(cluster.getFileSystem(0), testFile, BLOCK_SIZE * 8);

  List<List<Integer>> actualNumBlocks = getNumBlocksReport(0);
  assertEquals(cluster.getDataNodes().size(), actualNumBlocks.size());
  List<Integer> blocksOnFirstDN = actualNumBlocks.get(0);
  Collections.sort(blocksOnFirstDN);
  assertEquals(Arrays.asList(2, 2, 4, 4), blocksOnFirstDN);

  // Verify the second namespace also has the new volumes and they are empty.
  actualNumBlocks = getNumBlocksReport(1);
  assertEquals(4, actualNumBlocks.get(0).size());
  assertEquals(numNewVolumes,
      Collections.frequency(actualNumBlocks.get(0), 0));
}
 

/**
 * Test adding one volume on a running MiniDFSCluster with only one NameNode.
 */
@Test(timeout=60000)
public void testAddOneNewVolume()
    throws IOException, ReconfigurationException,
    InterruptedException, TimeoutException {
  startDFSCluster(1, 1);
  String bpid = cluster.getNamesystem().getBlockPoolId();
  final int numBlocks = 10;

  addVolumes(1);

  Path testFile = new Path("/test");
  createFile(testFile, numBlocks);

  List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
      cluster.getAllBlockReports(bpid);
  assertEquals(1, blockReports.size());  // 1 DataNode
  assertEquals(3, blockReports.get(0).size());  // 3 volumes

  // FSVolumeList uses Round-Robin block chooser by default. Thus the new
  // blocks should be evenly located in all volumes.
  int minNumBlocks = Integer.MAX_VALUE;
  int maxNumBlocks = Integer.MIN_VALUE;
  for (BlockListAsLongs blockList : blockReports.get(0).values()) {
    minNumBlocks = Math.min(minNumBlocks, blockList.getNumberOfBlocks());
    maxNumBlocks = Math.max(maxNumBlocks, blockList.getNumberOfBlocks());
  }
  assertTrue(Math.abs(maxNumBlocks - maxNumBlocks) <= 1);
  verifyFileLength(cluster.getFileSystem(), testFile, numBlocks);
}
 

@Override
protected void reconfigurePropertyImpl(String property, String newVal)
    throws ReconfigurationException {
  if (property.equals(TT_FAST_FETCH)) {
    this.fastFetch = Boolean.valueOf(newVal);
  } else if (property.equals(TT_OUTOFBAND_HEARBEAT)) {
    this.oobHeartbeatOnTaskCompletion = Boolean.valueOf(newVal);
  } else if (property.equals(TT_PROFILE_ALL_TASKS)) {
    this.profileAllTasks = Boolean.valueOf(newVal);
  }
}
 

@Test(timeout=60000)
public void testAddVolumesDuringWrite()
    throws IOException, InterruptedException, TimeoutException,
    ReconfigurationException {
  startDFSCluster(1, 1);
  String bpid = cluster.getNamesystem().getBlockPoolId();
  Path testFile = new Path("/test");
  createFile(testFile, 4);  // Each volume has 2 blocks.

  addVolumes(2);

  // Continue to write the same file, thus the new volumes will have blocks.
  DFSTestUtil.appendFile(cluster.getFileSystem(), testFile, BLOCK_SIZE * 8);
  verifyFileLength(cluster.getFileSystem(), testFile, 8 + 4);
  // After appending data, there should be [2, 2, 4, 4] blocks in each volume
  // respectively.
  List<Integer> expectedNumBlocks = Arrays.asList(2, 2, 4, 4);

  List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
      cluster.getAllBlockReports(bpid);
  assertEquals(1, blockReports.size());  // 1 DataNode
  assertEquals(4, blockReports.get(0).size());  // 4 volumes
  Map<DatanodeStorage, BlockListAsLongs> dnReport =
      blockReports.get(0);
  List<Integer> actualNumBlocks = new ArrayList<Integer>();
  for (BlockListAsLongs blockList : dnReport.values()) {
    actualNumBlocks.add(blockList.getNumberOfBlocks());
  }
  Collections.sort(actualNumBlocks);
  assertEquals(expectedNumBlocks, actualNumBlocks);
}
 

/** Test that a full block report is sent after hot swapping volumes */
@Test(timeout=100000)
public void testFullBlockReportAfterRemovingVolumes()
    throws IOException, ReconfigurationException {

  Configuration conf = new Configuration();
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);

  // Similar to TestTriggerBlockReport, set a really long value for
  // dfs.heartbeat.interval, so that incremental block reports and heartbeats
  // won't be sent during this test unless they're triggered
  // manually.
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10800000L);
  conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1080L);

  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  cluster.waitActive();

  final DataNode dn = cluster.getDataNodes().get(0);
  DatanodeProtocolClientSideTranslatorPB spy =
      DataNodeTestUtils.spyOnBposToNN(dn, cluster.getNameNode());

  // Remove a data dir from datanode
  File dataDirToKeep = new File(cluster.getDataDirectory(), "data1");
  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, dataDirToKeep.toString());

  // We should get 1 full report
  Mockito.verify(spy, timeout(60000).times(1)).blockReport(
      any(DatanodeRegistration.class),
      anyString(),
      any(StorageBlockReport[].class),
      any(BlockReportContext.class));
}
 

/**
 * Verify that {@link DataNode#checkDiskErrors()} removes all metadata in
 * DataNode upon a volume failure. Thus we can run reconfig on the same
 * configuration to reload the new volume on the same directory as the failed one.
 */
@Test(timeout=60000)
public void testDirectlyReloadAfterCheckDiskError()
    throws IOException, TimeoutException, InterruptedException,
    ReconfigurationException {
  startDFSCluster(1, 2);
  createFile(new Path("/test"), 32, (short)2);

  DataNode dn = cluster.getDataNodes().get(0);
  final String oldDataDir = dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY);
  File dirToFail = new File(cluster.getDataDirectory(), "data1");

  FsVolumeImpl failedVolume = getVolume(dn, dirToFail);
  assertTrue("No FsVolume was found for " + dirToFail,
      failedVolume != null);
  long used = failedVolume.getDfsUsed();

  DataNodeTestUtils.injectDataDirFailure(dirToFail);
  // Call and wait DataNode to detect disk failure.
  long lastDiskErrorCheck = dn.getLastDiskErrorCheck();
  dn.checkDiskErrorAsync();
  while (dn.getLastDiskErrorCheck() == lastDiskErrorCheck) {
    Thread.sleep(100);
  }

  createFile(new Path("/test1"), 32, (short)2);
  assertEquals(used, failedVolume.getDfsUsed());

  DataNodeTestUtils.restoreDataDirFromFailure(dirToFail);
  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, oldDataDir);

  createFile(new Path("/test2"), 32, (short)2);
  FsVolumeImpl restoredVolume = getVolume(dn, dirToFail);
  assertTrue(restoredVolume != null);
  assertTrue(restoredVolume != failedVolume);
  // More data has been written to this volume.
  assertTrue(restoredVolume.getDfsUsed() > used);
}
 

@Test(timeout=60000)
public void testAddBackRemovedVolume()
    throws IOException, TimeoutException, InterruptedException,
    ReconfigurationException {
  startDFSCluster(1, 2);
  // Create some data on every volume.
  createFile(new Path("/test"), 32);

  DataNode dn = cluster.getDataNodes().get(0);
  Configuration conf = dn.getConf();
  String oldDataDir = conf.get(DFS_DATANODE_DATA_DIR_KEY);
  String keepDataDir = oldDataDir.split(",")[0];
  String removeDataDir = oldDataDir.split(",")[1];

  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, keepDataDir);
  for (int i = 0; i < cluster.getNumNameNodes(); i++) {
    String bpid = cluster.getNamesystem(i).getBlockPoolId();
    BlockPoolSliceStorage bpsStorage =
        dn.getStorage().getBPStorage(bpid);
    // Make sure that there is no block pool level storage under removeDataDir.
    for (int j = 0; j < bpsStorage.getNumStorageDirs(); j++) {
      Storage.StorageDirectory sd = bpsStorage.getStorageDir(j);
      assertFalse(sd.getRoot().getAbsolutePath().startsWith(
          new File(removeDataDir).getAbsolutePath()
      ));
    }
    assertEquals(dn.getStorage().getBPStorage(bpid).getNumStorageDirs(), 1);
  }

  // Bring the removed directory back. It only successes if all metadata about
  // this directory were removed from the previous step.
  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, oldDataDir);
}
 

@Test(timeout=180000)
public void testRemoveVolumeBeingWritten()
    throws InterruptedException, TimeoutException, ReconfigurationException,
    IOException, BrokenBarrierException {
  // test against removing volumes on the different DataNode on the pipeline.
  for (int i = 0; i < 3; i++) {
    testRemoveVolumeBeingWrittenForDatanode(i);
  }
}
 

@Test(timeout=60000)
public void testRemoveOneVolume()
    throws ReconfigurationException, InterruptedException, TimeoutException,
    IOException {
  startDFSCluster(1, 1);
  final short replFactor = 1;
  Path testFile = new Path("/test");
  createFile(testFile, 10, replFactor);

  DataNode dn = cluster.getDataNodes().get(0);
  Collection<String> oldDirs = getDataDirs(dn);
  String newDirs = oldDirs.iterator().next();  // Keep the first volume.
  dn.reconfigurePropertyImpl(
      DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
  assertFileLocksReleased(
    new ArrayList<String>(oldDirs).subList(1, oldDirs.size()));
  dn.scheduleAllBlockReport(0);

  try {
    DFSTestUtil.readFile(cluster.getFileSystem(), testFile);
    fail("Expect to throw BlockMissingException.");
  } catch (BlockMissingException e) {
    GenericTestUtils.assertExceptionContains("Could not obtain block", e);
  }

  Path newFile = new Path("/newFile");
  createFile(newFile, 6);

  String bpid = cluster.getNamesystem().getBlockPoolId();
  List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
      cluster.getAllBlockReports(bpid);
  assertEquals((int)replFactor, blockReports.size());

  BlockListAsLongs blocksForVolume1 =
      blockReports.get(0).values().iterator().next();
  // The first volume has half of the testFile and full of newFile.
  assertEquals(10 / 2 + 6, blocksForVolume1.getNumberOfBlocks());
}
 

@Test(timeout=60000)
public void testAddVolumesToFederationNN()
    throws IOException, TimeoutException, InterruptedException,
    ReconfigurationException {
  // Starts a Cluster with 2 NameNode and 3 DataNodes. Each DataNode has 2
  // volumes.
  final int numNameNodes = 2;
  final int numDataNodes = 1;
  startDFSCluster(numNameNodes, numDataNodes);
  Path testFile = new Path("/test");
  // Create a file on the first namespace with 4 blocks.
  createFile(0, testFile, 4);
  // Create a file on the second namespace with 4 blocks.
  createFile(1, testFile, 4);

  // Add 2 volumes to the first DataNode.
  final int numNewVolumes = 2;
  addVolumes(numNewVolumes);

  // Append to the file on the first namespace.
  DFSTestUtil.appendFile(cluster.getFileSystem(0), testFile, BLOCK_SIZE * 8);

  List<List<Integer>> actualNumBlocks = getNumBlocksReport(0);
  assertEquals(cluster.getDataNodes().size(), actualNumBlocks.size());
  List<Integer> blocksOnFirstDN = actualNumBlocks.get(0);
  Collections.sort(blocksOnFirstDN);
  assertEquals(Arrays.asList(2, 2, 4, 4), blocksOnFirstDN);

  // Verify the second namespace also has the new volumes and they are empty.
  actualNumBlocks = getNumBlocksReport(1);
  assertEquals(4, actualNumBlocks.get(0).size());
  assertEquals(numNewVolumes,
      Collections.frequency(actualNumBlocks.get(0), 0));
}
 

@Test(timeout=60000)
public void testAddVolumesDuringWrite()
    throws IOException, InterruptedException, TimeoutException,
    ReconfigurationException {
  startDFSCluster(1, 1);
  String bpid = cluster.getNamesystem().getBlockPoolId();
  Path testFile = new Path("/test");
  createFile(testFile, 4);  // Each volume has 2 blocks.

  addVolumes(2);

  // Continue to write the same file, thus the new volumes will have blocks.
  DFSTestUtil.appendFile(cluster.getFileSystem(), testFile, BLOCK_SIZE * 8);
  verifyFileLength(cluster.getFileSystem(), testFile, 8 + 4);
  // After appending data, there should be [2, 2, 4, 4] blocks in each volume
  // respectively.
  List<Integer> expectedNumBlocks = Arrays.asList(2, 2, 4, 4);

  List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
      cluster.getAllBlockReports(bpid);
  assertEquals(1, blockReports.size());  // 1 DataNode
  assertEquals(4, blockReports.get(0).size());  // 4 volumes
  Map<DatanodeStorage, BlockListAsLongs> dnReport =
      blockReports.get(0);
  List<Integer> actualNumBlocks = new ArrayList<Integer>();
  for (BlockListAsLongs blockList : dnReport.values()) {
    actualNumBlocks.add(blockList.getNumberOfBlocks());
  }
  Collections.sort(actualNumBlocks);
  assertEquals(expectedNumBlocks, actualNumBlocks);
}
 

/**
 * Test adding one volume on a running MiniDFSCluster with only one NameNode.
 */
@Test(timeout=60000)
public void testAddOneNewVolume()
    throws IOException, ReconfigurationException,
    InterruptedException, TimeoutException {
  startDFSCluster(1, 1);
  String bpid = cluster.getNamesystem().getBlockPoolId();
  final int numBlocks = 10;

  addVolumes(1);

  Path testFile = new Path("/test");
  createFile(testFile, numBlocks);

  List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
      cluster.getAllBlockReports(bpid);
  assertEquals(1, blockReports.size());  // 1 DataNode
  assertEquals(3, blockReports.get(0).size());  // 3 volumes

  // FSVolumeList uses Round-Robin block chooser by default. Thus the new
  // blocks should be evenly located in all volumes.
  int minNumBlocks = Integer.MAX_VALUE;
  int maxNumBlocks = Integer.MIN_VALUE;
  for (BlockListAsLongs blockList : blockReports.get(0).values()) {
    minNumBlocks = Math.min(minNumBlocks, blockList.getNumberOfBlocks());
    maxNumBlocks = Math.max(maxNumBlocks, blockList.getNumberOfBlocks());
  }
  assertTrue(Math.abs(maxNumBlocks - maxNumBlocks) <= 1);
  verifyFileLength(cluster.getFileSystem(), testFile, numBlocks);
}
 

@Test(timeout=60000)
public void testRemoveOneVolume()
    throws ReconfigurationException, InterruptedException, TimeoutException,
    IOException {
  startDFSCluster(1, 1);
  final short replFactor = 1;
  Path testFile = new Path("/test");
  createFile(testFile, 10, replFactor);

  DataNode dn = cluster.getDataNodes().get(0);
  Collection<String> oldDirs = getDataDirs(dn);
  String newDirs = oldDirs.iterator().next();  // Keep the first volume.
  dn.reconfigurePropertyImpl(
      DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
  assertFileLocksReleased(
    new ArrayList<String>(oldDirs).subList(1, oldDirs.size()));
  dn.scheduleAllBlockReport(0);

  try {
    DFSTestUtil.readFile(cluster.getFileSystem(), testFile);
    fail("Expect to throw BlockMissingException.");
  } catch (BlockMissingException e) {
    GenericTestUtils.assertExceptionContains("Could not obtain block", e);
  }

  Path newFile = new Path("/newFile");
  createFile(newFile, 6);

  String bpid = cluster.getNamesystem().getBlockPoolId();
  List<Map<DatanodeStorage, BlockListAsLongs>> blockReports =
      cluster.getAllBlockReports(bpid);
  assertEquals((int)replFactor, blockReports.size());

  BlockListAsLongs blocksForVolume1 =
      blockReports.get(0).values().iterator().next();
  // The first volume has half of the testFile and full of newFile.
  assertEquals(10 / 2 + 6, blocksForVolume1.getNumberOfBlocks());
}
 

/** Test that a full block report is sent after hot swapping volumes */
@Test(timeout=100000)
public void testFullBlockReportAfterRemovingVolumes()
    throws IOException, ReconfigurationException {

  Configuration conf = new Configuration();
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);

  // Similar to TestTriggerBlockReport, set a really long value for
  // dfs.heartbeat.interval, so that incremental block reports and heartbeats
  // won't be sent during this test unless they're triggered
  // manually.
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10800000L);
  conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1080L);

  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  cluster.waitActive();

  final DataNode dn = cluster.getDataNodes().get(0);
  DatanodeProtocolClientSideTranslatorPB spy =
      DataNodeTestUtils.spyOnBposToNN(dn, cluster.getNameNode());

  // Remove a data dir from datanode
  File dataDirToKeep = new File(cluster.getDataDirectory(), "data1");
  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, dataDirToKeep.toString());

  // We should get 1 full report
  Mockito.verify(spy, timeout(60000).times(1)).blockReport(
      any(DatanodeRegistration.class),
      anyString(),
      any(StorageBlockReport[].class),
      any(BlockReportContext.class));
}
 

/**
 * Verify that {@link DataNode#checkDiskErrors()} removes all metadata in
 * DataNode upon a volume failure. Thus we can run reconfig on the same
 * configuration to reload the new volume on the same directory as the failed one.
 */
@Test(timeout=60000)
public void testDirectlyReloadAfterCheckDiskError()
    throws IOException, TimeoutException, InterruptedException,
    ReconfigurationException {
  startDFSCluster(1, 2);
  createFile(new Path("/test"), 32, (short)2);

  DataNode dn = cluster.getDataNodes().get(0);
  final String oldDataDir = dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY);
  File dirToFail = new File(cluster.getDataDirectory(), "data1");

  FsVolumeImpl failedVolume = getVolume(dn, dirToFail);
  assertTrue("No FsVolume was found for " + dirToFail,
      failedVolume != null);
  long used = failedVolume.getDfsUsed();

  DataNodeTestUtils.injectDataDirFailure(dirToFail);
  // Call and wait DataNode to detect disk failure.
  long lastDiskErrorCheck = dn.getLastDiskErrorCheck();
  dn.checkDiskErrorAsync();
  while (dn.getLastDiskErrorCheck() == lastDiskErrorCheck) {
    Thread.sleep(100);
  }

  createFile(new Path("/test1"), 32, (short)2);
  assertEquals(used, failedVolume.getDfsUsed());

  DataNodeTestUtils.restoreDataDirFromFailure(dirToFail);
  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, oldDataDir);

  createFile(new Path("/test2"), 32, (short)2);
  FsVolumeImpl restoredVolume = getVolume(dn, dirToFail);
  assertTrue(restoredVolume != null);
  assertTrue(restoredVolume != failedVolume);
  // More data has been written to this volume.
  assertTrue(restoredVolume.getDfsUsed() > used);
}
 

@Test(timeout=60000)
public void testAddBackRemovedVolume()
    throws IOException, TimeoutException, InterruptedException,
    ReconfigurationException {
  startDFSCluster(1, 2);
  // Create some data on every volume.
  createFile(new Path("/test"), 32);

  DataNode dn = cluster.getDataNodes().get(0);
  Configuration conf = dn.getConf();
  String oldDataDir = conf.get(DFS_DATANODE_DATA_DIR_KEY);
  String keepDataDir = oldDataDir.split(",")[0];
  String removeDataDir = oldDataDir.split(",")[1];

  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, keepDataDir);
  for (int i = 0; i < cluster.getNumNameNodes(); i++) {
    String bpid = cluster.getNamesystem(i).getBlockPoolId();
    BlockPoolSliceStorage bpsStorage =
        dn.getStorage().getBPStorage(bpid);
    // Make sure that there is no block pool level storage under removeDataDir.
    for (int j = 0; j < bpsStorage.getNumStorageDirs(); j++) {
      Storage.StorageDirectory sd = bpsStorage.getStorageDir(j);
      assertFalse(sd.getRoot().getAbsolutePath().startsWith(
          new File(removeDataDir).getAbsolutePath()
      ));
    }
    assertEquals(dn.getStorage().getBPStorage(bpid).getNumStorageDirs(), 1);
  }

  // Bring the removed directory back. It only successes if all metadata about
  // this directory were removed from the previous step.
  dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, oldDataDir);
}
 

@Test(timeout=180000)
public void testRemoveVolumeBeingWritten()
    throws InterruptedException, TimeoutException, ReconfigurationException,
    IOException, BrokenBarrierException {
  // test against removing volumes on the different DataNode on the pipeline.
  for (int i = 0; i < 3; i++) {
    testRemoveVolumeBeingWrittenForDatanode(i);
  }
}
 

@Test
public void testAddVolumeFailures() throws IOException {
  startDFSCluster(1, 1);
  final String dataDir = cluster.getDataDirectory();

  DataNode dn = cluster.getDataNodes().get(0);
  List<String> newDirs = Lists.newArrayList();
  final int NUM_NEW_DIRS = 4;
  for (int i = 0; i < NUM_NEW_DIRS; i++) {
    File newVolume = new File(dataDir, "new_vol" + i);
    newDirs.add(newVolume.toString());
    if (i % 2 == 0) {
      // Make addVolume() fail.
      newVolume.createNewFile();
    }
  }

  String newValue = dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY) + "," +
      Joiner.on(",").join(newDirs);
  try {
    dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, newValue);
    fail("Expect to throw IOException.");
  } catch (ReconfigurationException e) {
    String errorMessage = e.getCause().getMessage();
    String messages[] = errorMessage.split("\\r?\\n");
    assertEquals(2, messages.length);
    assertThat(messages[0], containsString("new_vol0"));
    assertThat(messages[1], containsString("new_vol2"));
  }

  // Make sure that vol0 and vol2's metadata are not left in memory.
  FsDatasetSpi<?> dataset = dn.getFSDataset();
  for (FsVolumeSpi volume : dataset.getVolumes()) {
    assertThat(volume.getBasePath(), is(not(anyOf(
        is(newDirs.get(0)), is(newDirs.get(2))))));
  }
  DataStorage storage = dn.getStorage();
  for (int i = 0; i < storage.getNumStorageDirs(); i++) {
    Storage.StorageDirectory sd = storage.getStorageDir(i);
    assertThat(sd.getRoot().toString(),
        is(not(anyOf(is(newDirs.get(0)), is(newDirs.get(2))))));
  }

  // The newly effective conf does not have vol0 and vol2.
  String[] effectiveVolumes =
      dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY).split(",");
  assertEquals(4, effectiveVolumes.length);
  for (String ev : effectiveVolumes) {
    assertThat(StorageLocation.parse(ev).getFile().getCanonicalPath(),
        is(not(anyOf(is(newDirs.get(0)), is(newDirs.get(2))))));
  }
}
 

@Test
public void testAddVolumeFailures() throws IOException {
  startDFSCluster(1, 1);
  final String dataDir = cluster.getDataDirectory();

  DataNode dn = cluster.getDataNodes().get(0);
  List<String> newDirs = Lists.newArrayList();
  final int NUM_NEW_DIRS = 4;
  for (int i = 0; i < NUM_NEW_DIRS; i++) {
    File newVolume = new File(dataDir, "new_vol" + i);
    newDirs.add(newVolume.toString());
    if (i % 2 == 0) {
      // Make addVolume() fail.
      newVolume.createNewFile();
    }
  }

  String newValue = dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY) + "," +
      Joiner.on(",").join(newDirs);
  try {
    dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, newValue);
    fail("Expect to throw IOException.");
  } catch (ReconfigurationException e) {
    String errorMessage = e.getCause().getMessage();
    String messages[] = errorMessage.split("\\r?\\n");
    assertEquals(2, messages.length);
    assertThat(messages[0], containsString("new_vol0"));
    assertThat(messages[1], containsString("new_vol2"));
  }

  // Make sure that vol0 and vol2's metadata are not left in memory.
  FsDatasetSpi<?> dataset = dn.getFSDataset();
  for (FsVolumeSpi volume : dataset.getVolumes()) {
    assertThat(volume.getBasePath(), is(not(anyOf(
        is(newDirs.get(0)), is(newDirs.get(2))))));
  }
  DataStorage storage = dn.getStorage();
  for (int i = 0; i < storage.getNumStorageDirs(); i++) {
    Storage.StorageDirectory sd = storage.getStorageDir(i);
    assertThat(sd.getRoot().toString(),
        is(not(anyOf(is(newDirs.get(0)), is(newDirs.get(2))))));
  }

  // The newly effective conf does not have vol0 and vol2.
  String[] effectiveVolumes =
      dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY).split(",");
  assertEquals(4, effectiveVolumes.length);
  for (String ev : effectiveVolumes) {
    assertThat(StorageLocation.parse(ev).getFile().getCanonicalPath(),
        is(not(anyOf(is(newDirs.get(0)), is(newDirs.get(2))))));
  }
}
 

/**
 * Test the case that remove a data volume on a particular DataNode when the
 * volume is actively being written.
 * @param dataNodeIdx the index of the DataNode to remove a volume.
 */
private void testRemoveVolumeBeingWrittenForDatanode(int dataNodeIdx)
    throws IOException, ReconfigurationException, TimeoutException,
    InterruptedException, BrokenBarrierException {
  // Starts DFS cluster with 3 DataNodes to form a pipeline.
  startDFSCluster(1, 3);

  final short REPLICATION = 3;
  final DataNode dn = cluster.getDataNodes().get(dataNodeIdx);
  final FileSystem fs = cluster.getFileSystem();
  final Path testFile = new Path("/test");
  final long lastTimeDiskErrorCheck = dn.getLastDiskErrorCheck();

  FSDataOutputStream out = fs.create(testFile, REPLICATION);

  Random rb = new Random(0);
  byte[] writeBuf = new byte[BLOCK_SIZE / 2];  // half of the block.
  rb.nextBytes(writeBuf);
  out.write(writeBuf);
  out.hflush();

  // Make FsDatasetSpi#finalizeBlock a time-consuming operation. So if the
  // BlockReceiver releases volume reference before finalizeBlock(), the blocks
  // on the volume will be removed, and finalizeBlock() throws IOE.
  final FsDatasetSpi<? extends FsVolumeSpi> data = dn.data;
  dn.data = Mockito.spy(data);
  doAnswer(new Answer<Object>() {
        public Object answer(InvocationOnMock invocation)
            throws IOException, InterruptedException {
          Thread.sleep(1000);
          // Bypass the argument to FsDatasetImpl#finalizeBlock to verify that
          // the block is not removed, since the volume reference should not
          // be released at this point.
          data.finalizeBlock((ExtendedBlock) invocation.getArguments()[0]);
          return null;
        }
      }).when(dn.data).finalizeBlock(any(ExtendedBlock.class));

  final CyclicBarrier barrier = new CyclicBarrier(2);

  List<String> oldDirs = getDataDirs(dn);
  final String newDirs = oldDirs.get(1);  // Remove the first volume.
  final List<Exception> exceptions = new ArrayList<>();
  Thread reconfigThread = new Thread() {
    public void run() {
      try {
        barrier.await();
        dn.reconfigurePropertyImpl(
            DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
      } catch (ReconfigurationException |
          InterruptedException |
          BrokenBarrierException e) {
        exceptions.add(e);
      }
    }
  };
  reconfigThread.start();

  barrier.await();
  rb.nextBytes(writeBuf);
  out.write(writeBuf);
  out.hflush();
  out.close();

  reconfigThread.join();

  // Verify the file has sufficient replications.
  DFSTestUtil.waitReplication(fs, testFile, REPLICATION);
  // Read the content back
  byte[] content = DFSTestUtil.readFileBuffer(fs, testFile);
  assertEquals(BLOCK_SIZE, content.length);

  // If an IOException thrown from BlockReceiver#run, it triggers
  // DataNode#checkDiskError(). So we can test whether checkDiskError() is called,
  // to see whether there is IOException in BlockReceiver#run().
  assertEquals(lastTimeDiskErrorCheck, dn.getLastDiskErrorCheck());

  if (!exceptions.isEmpty()) {
    throw new IOException(exceptions.get(0).getCause());
  }
}
 

/**
 * Test that we can change the block placement policy through the
 * reconfigurable API.
 */
@Test
public void testChangeBlockPlacementPolicy()
  throws IOException, ReconfigurationException {
  AtomicInteger callCounter = new AtomicInteger(0);
  MockPlacementPolicy.setCallCounter(callCounter);

  DFSTestUtil util = new DFSTestUtil("", 2, 1, 512);

  // write some files with the default block placement policy
  util.createFiles(fs, "/reconfdat1", (short) 3);
  util.waitReplication(fs, "/reconfdat1", (short) 3);

  assertTrue("calls already made to MockPlacementPolicy",
             callCounter.get() == 0);

  // switch over to the mock placement policy
  cluster.getNameNode().reconfigureProperty("dfs.block.replicator.classname",
                                            "org.apache.hadoop.hdfs.server." +
                                            "namenode." +
                                            "TestNameNodeReconfigure$" +
                                            "MockPlacementPolicy");

  // write some files with the mock placement policy
  util.createFiles(fs, "/reconfdat2", (short) 3);
  util.waitReplication(fs, "/reconfdat2", (short) 3);

  int callsMade1 = callCounter.get();
  
  // check that calls were made to mock placement policy
  assertTrue("no calls made to MockPlacementPolicy",
             callsMade1 > 0);
  LOG.info("" + callsMade1 + " calls made to MockPlacementPolicy");

  // now try to change it to a non-existent class
  try {
    cluster.getNameNode().
      reconfigureProperty("dfs.block.replicator.classname",
                          "does.not.exist");
    fail("ReconfigurationException expected");
  } catch (RuntimeException expected) {
    assertTrue("exception should have cause", expected.getCause() != null);
    assertTrue("exception's cause should have cause",
               expected.getCause().getCause() != null);
    assertTrue("ClassNotFoundException expected but got " +
               expected.getCause().getCause().getClass().getCanonicalName(),
               expected.getCause().getCause() instanceof
               ClassNotFoundException);
  }

  // write some files, they should still go to the mock placemeny policy
  util.createFiles(fs, "/reconfdat3", (short) 3);
  util.waitReplication(fs, "/reconfdat3", (short) 3);

  int callsMade2 = callCounter.get();

  // check that more calls were made to mock placement policy
  assertTrue("no calls made to MockPlacementPolicy",
             callsMade2 > callsMade1);
  LOG.info("" + (callsMade2 - callsMade1) +
    " calls made to MockPlacementPolicy");

  // now revert back to the default policy
  cluster.getNameNode().reconfigureProperty("dfs.block.replicator.classname",
                                            null);

  // write some files with the default block placement policy
  util.createFiles(fs, "/reconfdat4", (short) 3);
  util.waitReplication(fs, "/reconfdat4", (short) 3);

  // make sure that no more calls were made to mock placement policy
  assertTrue("more calls made to MockPlacementPolicy",
             callCounter.get() == callsMade2);

  util.cleanup(fs, "/reconfdat1");
  util.cleanup(fs, "/reconfdat2");
  util.cleanup(fs, "/reconfdat3");
  util.cleanup(fs, "/reconfdat4");
}