Java Code Examples for org.apache.hadoop.hdfs.MiniDFSCluster#startDataNodes()

private void testBalancerDefaultConstructor(Configuration conf, long[] capacities,
    String[] racks, long newCapacity, String newRack)
    throws Exception {
  int numOfDatanodes = capacities.length;
  assertEquals(numOfDatanodes, racks.length);
  cluster = new MiniDFSCluster(0, conf, capacities.length, true, true, null, 
      racks, capacities);
  try {
    cluster.waitActive();
    client = DFSClient.createNamenode(conf);

    long totalCapacity = 0L;
    for (long capacity : capacities) {
      totalCapacity += capacity;
    }
    // fill up the cluster to be 30% full
    long totalUsedSpace = totalCapacity * 3 / 10;
    createFile(totalUsedSpace / numOfDatanodes, (short) numOfDatanodes);
    // start up an empty node with the same capacity and on the same rack
    cluster.startDataNodes(conf, 1, true, null, new String[] { newRack },
        new long[] { newCapacity });
    totalCapacity += newCapacity;
    // run balancer and validate results
    runBalancer(conf, totalUsedSpace, totalCapacity);
  } finally {
    cluster.shutdown();
  }
}
 

@Test(timeout = 20000)
public void testClusterIdMismatchAtStartupWithHA() throws Exception {
  MiniDFSNNTopology top = new MiniDFSNNTopology()
    .addNameservice(new MiniDFSNNTopology.NSConf("ns1")
      .addNN(new MiniDFSNNTopology.NNConf("nn0"))
      .addNN(new MiniDFSNNTopology.NNConf("nn1")))
    .addNameservice(new MiniDFSNNTopology.NSConf("ns2")
      .addNN(new MiniDFSNNTopology.NNConf("nn2").setClusterId("bad-cid"))
      .addNN(new MiniDFSNNTopology.NNConf("nn3").setClusterId("bad-cid")));

  top.setFederation(true);

  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(top)
      .numDataNodes(0).build();
  
  try {
    cluster.startDataNodes(conf, 1, true, null, null);
    // let the initialization be complete
    Thread.sleep(10000);
    DataNode dn = cluster.getDataNodes().get(0);
    assertTrue("Datanode should be running", dn.isDatanodeUp());
    assertEquals("Only one BPOfferService should be running", 1,
        dn.getAllBpOs().length);
  } finally {
    cluster.shutdown();
  }
}
 

@Test
public void testSufficientlyReplBlocksUsesNewRack() throws Exception {
  Configuration conf = getConf();
  final short REPLICATION_FACTOR = 3;
  final Path filePath = new Path("/testFile");
  // All datanodes are on the same rack
  String racks[] = {"/rack1", "/rack1", "/rack1"};
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
    .numDataNodes(racks.length).racks(racks).build();

  try {
    // Create a file with one block with a replication factor of 3
    final FileSystem fs = cluster.getFileSystem();
    DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
    ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
    DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0);

    // Add a new datanode on a different rack
    String newRacks[] = {"/rack2"};
    cluster.startDataNodes(conf, 1, true, null, newRacks);
    cluster.waitActive();

    DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
  } finally {
    cluster.shutdown();
  }
}
 

@Test
public void testUnderReplicatedUsesNewRacks() throws Exception {
  Configuration conf = getConf();
  short REPLICATION_FACTOR = 3;
  final Path filePath = new Path("/testFile");
  // All datanodes are on the same rack
  String racks[] = {"/rack1", "/rack1", "/rack1", "/rack1", "/rack1"};
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
    .numDataNodes(racks.length).racks(racks).build();
  final FSNamesystem ns = cluster.getNameNode().getNamesystem();

  try {
    // Create a file with one block
    final FileSystem fs = cluster.getFileSystem();
    DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
    ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
    DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0);
    
    // Add new datanodes on a different rack and increase the
    // replication factor so the block is underreplicated and make
    // sure at least one of the hosts on the new rack is used. 
    String newRacks[] = {"/rack2", "/rack2"};
    cluster.startDataNodes(conf, 2, true, null, newRacks);
    REPLICATION_FACTOR = 5;
    NameNodeAdapter.setReplication(ns, "/testFile", REPLICATION_FACTOR);

    DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
  } finally {
    cluster.shutdown();
  }
}
 

@Test
public void testTransferRbw() throws Exception {
  final HdfsConfiguration conf = new HdfsConfiguration();
  final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf
      ).numDataNodes(REPLICATION).build();
  try {
    cluster.waitActive();
    final DistributedFileSystem fs = cluster.getFileSystem();

    //create a file, write some data and leave it open. 
    final Path p = new Path("/foo");
    final int size = (1 << 16) + RAN.nextInt(1 << 16);
    LOG.info("size = " + size);
    final FSDataOutputStream out = fs.create(p, REPLICATION);
    final byte[] bytes = new byte[1024];
    for(int remaining = size; remaining > 0; ) {
      RAN.nextBytes(bytes);
      final int len = bytes.length < remaining? bytes.length: remaining;
      out.write(bytes, 0, len);
      out.hflush();
      remaining -= len;
    }

    //get the RBW
    final ReplicaBeingWritten oldrbw;
    final DataNode newnode;
    final DatanodeInfo newnodeinfo;
    final String bpid = cluster.getNamesystem().getBlockPoolId();
    {
      final DataNode oldnode = cluster.getDataNodes().get(0);
      oldrbw = getRbw(oldnode, bpid);
      LOG.info("oldrbw = " + oldrbw);
      
      //add a datanode
      cluster.startDataNodes(conf, 1, true, null, null);
      newnode = cluster.getDataNodes().get(REPLICATION);
      
      final DatanodeInfo oldnodeinfo;
      {
        final DatanodeInfo[] datatnodeinfos = cluster.getNameNodeRpc(
            ).getDatanodeReport(DatanodeReportType.LIVE);
        Assert.assertEquals(2, datatnodeinfos.length);
        int i = 0;
        for(DatanodeRegistration dnReg = newnode.getDNRegistrationForBP(bpid);
            i < datatnodeinfos.length && !datatnodeinfos[i].equals(dnReg); i++);
        Assert.assertTrue(i < datatnodeinfos.length);
        newnodeinfo = datatnodeinfos[i];
        oldnodeinfo = datatnodeinfos[1 - i];
      }
      
      //transfer RBW
      final ExtendedBlock b = new ExtendedBlock(bpid, oldrbw.getBlockId(), oldrbw.getBytesAcked(),
          oldrbw.getGenerationStamp());
      final BlockOpResponseProto s = DFSTestUtil.transferRbw(
          b, DFSClientAdapter.getDFSClient(fs), oldnodeinfo, newnodeinfo);
      Assert.assertEquals(Status.SUCCESS, s.getStatus());
    }

    //check new rbw
    final ReplicaBeingWritten newrbw = getRbw(newnode, bpid);
    LOG.info("newrbw = " + newrbw);
    Assert.assertEquals(oldrbw.getBlockId(), newrbw.getBlockId());
    Assert.assertEquals(oldrbw.getGenerationStamp(), newrbw.getGenerationStamp());
    Assert.assertEquals(oldrbw.getVisibleLength(), newrbw.getVisibleLength());

    LOG.info("DONE");
  } finally {
    cluster.shutdown();
  }
}
 

public void testNodeCount() throws Exception {
  // start a mini dfs cluster of 2 nodes
  final Configuration conf = new Configuration();
  conf.setInt("dfs.replication.interval", 10);
  final short REPLICATION_FACTOR = (short)2;
  final MiniDFSCluster cluster = 
    new MiniDFSCluster(conf, REPLICATION_FACTOR, true, null);
  try {
    final FSNamesystem namesystem = cluster.getNameNode().namesystem;
    final FileSystem fs = cluster.getFileSystem();
    
    // populate the cluster with a one block file
    final Path FILE_PATH = new Path("/testfile");
    DFSTestUtil.createFile(fs, FILE_PATH, 1L, REPLICATION_FACTOR, 1L);
    DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR);
    Block block = DFSTestUtil.getFirstBlock(fs, FILE_PATH);

    // keep a copy of all datanode descriptor
    DatanodeDescriptor[] datanodes = (DatanodeDescriptor[])
       namesystem.heartbeats.toArray(new DatanodeDescriptor[REPLICATION_FACTOR]);
    
    // start two new nodes
    cluster.startDataNodes(conf, 2, true, null, null);
    cluster.waitActive(false);
    
    LOG.info("Bringing down first DN");
    // bring down first datanode
    DatanodeDescriptor datanode = datanodes[0];
    DataNodeProperties dnprop = cluster.stopDataNode(datanode.getName());
    // make sure that NN detects that the datanode is down
    synchronized (namesystem.heartbeats) {
      datanode.setLastUpdate(0); // mark it dead
      namesystem.heartbeatCheck();
    }

    LOG.info("Waiting for block to be replicated");
    // the block will be replicated
    DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR);

    LOG.info("Restarting first datanode");
    // restart the first datanode
    cluster.restartDataNode(dnprop);
    cluster.waitActive(false);

    LOG.info("Waiting for excess replicas to be detected");
    
    // check if excessive replica is detected
    waitForExcessReplicasToChange(namesystem, block, 1);

    LOG.info("Finding a non-excess node");

    // find out a non-excess node
    Iterator<DatanodeDescriptor> iter = namesystem.blocksMap.nodeIterator(block);
    DatanodeDescriptor nonExcessDN = null;
    while (iter.hasNext()) {
      DatanodeDescriptor dn = iter.next();
      Collection<Block> blocks = namesystem.excessReplicateMap.get(dn.getStorageID());
      if (blocks == null || !blocks.contains(block) ) {
        nonExcessDN = dn;
        break;
      }
    }
    assertTrue(nonExcessDN!=null);

    LOG.info("Stopping non-excess node: " + nonExcessDN);
    // bring down non excessive datanode
    dnprop = cluster.stopDataNode(nonExcessDN.getName());
    // make sure that NN detects that the datanode is down
    synchronized (namesystem.heartbeats) {
      nonExcessDN.setLastUpdate(0); // mark it dead
      namesystem.heartbeatCheck();
    }
    
    LOG.info("Waiting for live replicas to hit repl factor");
    // The block should be replicated
    NumberReplicas num;
    do {
     namesystem.readLock();
     try {
       num = namesystem.countNodes(block);
     } finally {
       namesystem.readUnlock();
     }
    } while (num.liveReplicas() != REPLICATION_FACTOR);
    
    LOG.info("Restarting first DN");
    // restart the first datanode
    cluster.restartDataNode(dnprop);
    cluster.waitActive(false);
    // check if excessive replica is detected
    LOG.info("Waiting for excess replicas to be detected");
    waitForExcessReplicasToChange(namesystem, block, 2);
  } finally {
    cluster.shutdown();
  }
}
 

private void startCluster(int numHosts, int numTrackerPerHost, 
                          int numExcluded, Configuration conf) 
throws IOException {
  try {
    conf.setBoolean("dfs.replication.considerLoad", false);
    
    // prepare hosts info
    hosts = new String[numHosts];
    for (int i = 1; i <= numHosts; ++i) {
      hosts[i - 1] = getHostname(i);
    }
    
    // start dfs
    dfs = new MiniDFSCluster(conf, 1, true, null, hosts);
    dfs.waitActive();
    dfs.startDataNodes(conf, numHosts, true, null, null, hosts, null);
    dfs.waitActive();

    namenode = (dfs.getFileSystem()).getUri().getHost() + ":" + 
    (dfs.getFileSystem()).getUri().getPort(); 
    
    // create tracker hosts
    trackerHosts = new String[numHosts * numTrackerPerHost];
    for (int i = 1; i <= (numHosts * numTrackerPerHost); ++i) {
      trackerHosts[i - 1] = getHostname(i);
    }
    
    // start mini mr
    JobConf jtConf = new JobConf(conf);
    mr = new MiniMRCluster(0, 0, numHosts * numTrackerPerHost, namenode, 1, 
                           null, trackerHosts, null, jtConf, 
                           numExcluded * numTrackerPerHost);
    
    jt = mr.getJobTrackerRunner().getJobTracker();
    
    // check if trackers from all the desired hosts have connected
    Set<String> hostsSeen = new HashSet<String>();
    for (TaskTrackerStatus status : jt.taskTrackers()) {
      hostsSeen.add(status.getHost());
    }
    assertEquals("Not all hosts are up", numHosts - numExcluded, 
                 hostsSeen.size());
  } catch (IOException ioe) {
    stopCluster();
  }
}
 

public void testNodeCount() throws Exception {
  // start a mini dfs cluster of 2 nodes
  final Configuration conf = new Configuration();
  final short REPLICATION_FACTOR = (short)2;
  final MiniDFSCluster cluster = 
    new MiniDFSCluster(conf, REPLICATION_FACTOR, true, null);
  try {
    final FSNamesystem namesystem = cluster.getNameNode().namesystem;
    final FileSystem fs = cluster.getFileSystem();
    
    // populate the cluster with a one block file
    final Path FILE_PATH = new Path("/testfile");
    DFSTestUtil.createFile(fs, FILE_PATH, 1L, REPLICATION_FACTOR, 1L);
    DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR);
    Block block = DFSTestUtil.getFirstBlock(fs, FILE_PATH);

    // keep a copy of all datanode descriptor
    DatanodeDescriptor[] datanodes = (DatanodeDescriptor[])
       namesystem.heartbeats.toArray(new DatanodeDescriptor[REPLICATION_FACTOR]);
    
    // start two new nodes
    cluster.startDataNodes(conf, 2, true, null, null);
    cluster.waitActive();
    
    // bring down first datanode
    DatanodeDescriptor datanode = datanodes[0];
    DataNodeProperties dnprop = cluster.stopDataNode(datanode.getName());
    // make sure that NN detects that the datanode is down
    synchronized (namesystem.heartbeats) {
      datanode.setLastUpdate(0); // mark it dead
      namesystem.heartbeatCheck();
    }
    // the block will be replicated
    DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR);

    // restart the first datanode
    cluster.restartDataNode(dnprop);
    cluster.waitActive();
    
    // check if excessive replica is detected
    NumberReplicas num = null;
    do {
     synchronized (namesystem) {
       num = namesystem.countNodes(block);
     }
    } while (num.excessReplicas() == 0);
    
    // find out a non-excess node
    Iterator<DatanodeDescriptor> iter = namesystem.blocksMap.nodeIterator(block);
    DatanodeDescriptor nonExcessDN = null;
    while (iter.hasNext()) {
      DatanodeDescriptor dn = iter.next();
      Collection<Block> blocks = namesystem.excessReplicateMap.get(dn.getStorageID());
      if (blocks == null || !blocks.contains(block) ) {
        nonExcessDN = dn;
        break;
      }
    }
    assertTrue(nonExcessDN!=null);
    
    // bring down non excessive datanode
    dnprop = cluster.stopDataNode(nonExcessDN.getName());
    // make sure that NN detects that the datanode is down
    synchronized (namesystem.heartbeats) {
      nonExcessDN.setLastUpdate(0); // mark it dead
      namesystem.heartbeatCheck();
    }
    
    // The block should be replicated
    do {
      num = namesystem.countNodes(block);
    } while (num.liveReplicas() != REPLICATION_FACTOR);
    
    // restart the first datanode
    cluster.restartDataNode(dnprop);
    cluster.waitActive();
    
    // check if excessive replica is detected
    do {
     num = namesystem.countNodes(block);
    } while (num.excessReplicas() == 2);
  } finally {
    cluster.shutdown();
  }
}
 

/**
 * The test verifies the number of outstanding replication requests for a
 * given DN shouldn't exceed the limit set by configuration property
 * dfs.namenode.replication.max-streams-hard-limit.
 * The test does the followings:
 * 1. Create a mini cluster with 2 DNs. Set large heartbeat interval so that
 *    replication requests won't be picked by any DN right away.
 * 2. Create a file with 10 blocks and replication factor 2. Thus each
 *    of the 2 DNs have one replica of each block.
 * 3. Add a DN to the cluster for later replication.
 * 4. Remove a DN that has data.
 * 5. Ask BlockManager to compute the replication work. This will assign
 *    replication requests to the only DN that has data.
 * 6. Make sure the number of pending replication requests of that DN don't
 *    exceed the limit.
 * @throws Exception
 */
@Test(timeout=60000) // 1 min timeout
public void testNumberOfBlocksToBeReplicated() throws Exception {
  Configuration conf = new HdfsConfiguration();

  conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0);
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1);
  conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);

  // Large value to make sure the pending replication request can stay in
  // DatanodeDescriptor.replicateBlocks before test timeout.
  conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 100);

  // Make sure BlockManager can pull all blocks from UnderReplicatedBlocks via
  // chooseUnderReplicatedBlocks at once.
   conf.setInt(
      DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, 5);

  int NUM_OF_BLOCKS = 10;
  final short REP_FACTOR = 2;
  final String FILE_NAME = "/testFile";
  final Path FILE_PATH = new Path(FILE_NAME);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
          REP_FACTOR).build();
  try {
    // create a file with 10 blocks with a replication factor of 2
    final FileSystem fs = cluster.getFileSystem();
    DFSTestUtil.createFile(fs, FILE_PATH, NUM_OF_BLOCKS, REP_FACTOR, 1L);
    DFSTestUtil.waitReplication(fs, FILE_PATH, REP_FACTOR);

    cluster.startDataNodes(conf, 1, true, null, null, null, null);

    final BlockManager bm = cluster.getNamesystem().getBlockManager();
    ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, FILE_PATH);
    Iterator<DatanodeStorageInfo> storageInfos =
        bm.blocksMap.getStorages(b.getLocalBlock())
        .iterator();
    DatanodeDescriptor firstDn = storageInfos.next().getDatanodeDescriptor();
    DatanodeDescriptor secondDn = storageInfos.next().getDatanodeDescriptor();

    bm.getDatanodeManager().removeDatanode(firstDn);

    assertEquals(NUM_OF_BLOCKS, bm.getUnderReplicatedNotMissingBlocks());
    bm.computeDatanodeWork();


    assertTrue("The number of blocks to be replicated should be less than "
        + "or equal to " + bm.replicationStreamsHardLimit,
        secondDn.getNumberOfBlocksToBeReplicated()
        <= bm.replicationStreamsHardLimit);
  } finally {
    cluster.shutdown();
  }

}
 

/**
 * This test start a cluster, fill the DataNodes to be 30% full;
 * It then adds an empty node and start balancing.
 *
 * @param nNameNodes Number of NameNodes
 * @param capacities Capacities of the datanodes
 * @param racks Rack names
 * @param newCapacity the capacity of the new DataNode
 * @param newRack the rack for the new DataNode
 * @param conf Configuration
 */ 
private void runTest(final int nNameNodes, long[] capacities, String[] racks,
    long newCapacity, String newRack, Configuration conf) throws Exception {
  final int nDataNodes = capacities.length;
  LOG.info("nNameNodes=" + nNameNodes + ", nDataNodes=" + nDataNodes);
  Assert.assertEquals(nDataNodes, racks.length);

  LOG.info("RUN_TEST -1");
  final MiniDFSCluster cluster = new MiniDFSCluster
      .Builder(new Configuration(conf))
      .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(nNameNodes))
      .numDataNodes(nDataNodes)
      .racks(racks)
      .simulatedCapacities(capacities)
      .build();
  LOG.info("RUN_TEST 0");
  DFSTestUtil.setFederatedConfiguration(cluster, conf);

  try {
    cluster.waitActive();
    LOG.info("RUN_TEST 1");
    final Suite s = new Suite(cluster, nNameNodes, nDataNodes, conf);
    long totalCapacity = TestBalancer.sum(capacities);

    LOG.info("RUN_TEST 2");
    // fill up the cluster to be 30% full
    final long totalUsed = totalCapacity*3/10;
    final long size = (totalUsed/nNameNodes)/s.replication;
    for(int n = 0; n < nNameNodes; n++) {
      createFile(s, n, size);
    }

    LOG.info("RUN_TEST 3");
    // start up an empty node with the same capacity and on the same rack
    cluster.startDataNodes(conf, 1, true, null,
        new String[]{newRack}, new long[]{newCapacity});

    totalCapacity += newCapacity;

    LOG.info("RUN_TEST 4");
    // run RUN_TEST and validate results
    runBalancer(s, totalUsed, totalCapacity);
    LOG.info("RUN_TEST 5");
  } finally {
    cluster.shutdown();
  }
  LOG.info("RUN_TEST 6");
}
 

/**
 * Verify that a file can be read both from NameNode and BackupNode.
 */
@Test
public void testCanReadData() throws IOException {
  Path file1 = new Path("/fileToRead.dat");
  Configuration conf = new HdfsConfiguration();
  MiniDFSCluster cluster = null;
  FileSystem fileSys = null;
  BackupNode backup = null;
  try {
    // Start NameNode and BackupNode
    cluster = new MiniDFSCluster.Builder(conf)
                                .numDataNodes(0).format(true).build();
    fileSys = cluster.getFileSystem();
    long txid = cluster.getNameNodeRpc().getTransactionID();
    backup = startBackupNode(conf, StartupOption.BACKUP, 1);
    waitCheckpointDone(cluster, txid);

    // Setup dual NameNode configuration for DataNodes
    String rpcAddrKeyPreffix =
        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + ".bnCluster";
    String nnAddr = cluster.getNameNode().getNameNodeAddressHostPortString();
        conf.get(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
    String bnAddr = backup.getNameNodeAddressHostPortString();
    conf.set(DFSConfigKeys.DFS_NAMESERVICES, "bnCluster");
    conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, "bnCluster");
    conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + ".bnCluster",
        "nnActive, nnBackup");
    conf.set(rpcAddrKeyPreffix + ".nnActive", nnAddr);
    conf.set(rpcAddrKeyPreffix + ".nnBackup", bnAddr);
    cluster.startDataNodes(conf, 3, true, StartupOption.REGULAR, null);

    DFSTestUtil.createFile(
        fileSys, file1, 8192, (short)3, 0);

    // Read the same file from file systems pointing to NN and BN
    FileSystem bnFS = FileSystem.get(
        new Path("hdfs://" + bnAddr).toUri(), conf);
    String nnData = DFSTestUtil.readFile(fileSys, file1);
    String bnData = DFSTestUtil.readFile(bnFS, file1);
    assertEquals("Data read from BackupNode and NameNode is not the same.",
        nnData, bnData);
  } catch(IOException e) {
    LOG.error("Error in TestBackupNode: ", e);
    assertTrue(e.getLocalizedMessage(), false);
  } finally {
    if(fileSys != null) fileSys.close();
    if(backup != null) backup.stop();
    if(cluster != null) cluster.shutdown();
  }
}
 

/**
 * The test verifies that replica for deletion is chosen on a node,
 * with the oldest heartbeat, when this heartbeat is larger than the
 * tolerable heartbeat interval.
 * It creates a file with several blocks and replication 4.
 * The last DN is configured to send heartbeats rarely.
 * 
 * Test waits until the tolerable heartbeat interval expires, and reduces
 * replication of the file. All replica deletions should be scheduled for the
 * last node. No replicas will actually be deleted, since last DN doesn't
 * send heartbeats. 
 */
@Test
public void testChooseReplicaToDelete() throws Exception {
  MiniDFSCluster cluster = null;
  FileSystem fs = null;
  try {
    Configuration conf = new HdfsConfiguration();
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, SMALL_BLOCK_SIZE);
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    fs = cluster.getFileSystem();
    final FSNamesystem namesystem = cluster.getNamesystem();

    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 300);
    cluster.startDataNodes(conf, 1, true, null, null, null);
    DataNode lastDN = cluster.getDataNodes().get(3);
    DatanodeRegistration dnReg = DataNodeTestUtils.getDNRegistrationForBP(
        lastDN, namesystem.getBlockPoolId());
    String lastDNid = dnReg.getDatanodeUuid();

    final Path fileName = new Path("/foo2");
    DFSTestUtil.createFile(fs, fileName, SMALL_FILE_LENGTH, (short)4, 0L);
    DFSTestUtil.waitReplication(fs, fileName, (short)4);

    // Wait for tolerable number of heartbeats plus one
    DatanodeDescriptor nodeInfo = null;
    long lastHeartbeat = 0;
    long waitTime = DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT * 1000 *
      (DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_DEFAULT + 1);
    do {
      nodeInfo = namesystem.getBlockManager().getDatanodeManager()
          .getDatanode(dnReg);
      lastHeartbeat = nodeInfo.getLastUpdateMonotonic();
    } while (monotonicNow() - lastHeartbeat < waitTime);
    fs.setReplication(fileName, (short)3);

    BlockLocation locs[] = fs.getFileBlockLocations(
        fs.getFileStatus(fileName), 0, Long.MAX_VALUE);

    // All replicas for deletion should be scheduled on lastDN.
    // And should not actually be deleted, because lastDN does not heartbeat.
    namesystem.readLock();
    Collection<Block> dnBlocks = 
      namesystem.getBlockManager().excessReplicateMap.get(lastDNid);
    assertEquals("Replicas on node " + lastDNid + " should have been deleted",
        SMALL_FILE_LENGTH / SMALL_BLOCK_SIZE, dnBlocks.size());
    namesystem.readUnlock();
    for(BlockLocation location : locs)
      assertEquals("Block should still have 4 replicas",
          4, location.getNames().length);
  } finally {
    if(fs != null) fs.close();
    if(cluster != null) cluster.shutdown();
  }
}
 

public void testReplicationError() throws Exception {
  // bring up a cluster of 1
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
  cluster.waitActive();
  FileSystem fs = cluster.getFileSystem();
  
  try {
    // create a file of replication factor of 1
    final Path fileName = new Path("/test.txt");
    final int fileLen = 1;
    DFSTestUtil.createFile(fs, fileName, 1, (short)1, 1L);
    DFSTestUtil.waitReplication(fs, fileName, (short)1);

    // get the block belonged to the created file
    LocatedBlocks blocks = cluster.getNameNode().namesystem.getBlockLocations(
        fileName.toString(), 0, (long)fileLen);
    assertEquals(blocks.locatedBlockCount(), 1);
    LocatedBlock block = blocks.get(0);
    
    // bring up a second datanode
    cluster.startDataNodes(conf, 1, true, null, null);
    cluster.waitActive();
    final int sndNode = 1;
    DataNode datanode = cluster.getDataNodes().get(sndNode);
    
    // replicate the block to the second datanode
    InetSocketAddress target = datanode.getSelfAddr();
    Socket s = new Socket(target.getAddress(), target.getPort());
      //write the header.
    DataOutputStream out = new DataOutputStream(
        s.getOutputStream());

    out.writeShort( DataTransferProtocol.DATA_TRANSFER_VERSION );
    out.write( DataTransferProtocol.OP_WRITE_BLOCK );
    out.writeLong( block.getBlock().getBlockId());
    out.writeLong( block.getBlock().getGenerationStamp() );
    out.writeInt(1);
    out.writeBoolean( false );       // recovery flag
    Text.writeString( out, "" );
    out.writeBoolean(false); // Not sending src node information
    out.writeInt(0);
    
    // write check header
    out.writeByte( 1 );
    out.writeInt( 512 );

    out.flush();

    // close the connection before sending the content of the block
    out.close();
    
    // the temporary block & meta files should be deleted
    String dataDir = cluster.getDataDirectory();
    File dir1 = new File(new File(dataDir, "data"+(2*sndNode+1)), "tmp");
    File dir2 = new File(new File(dataDir, "data"+(2*sndNode+2)), "tmp");
    while (dir1.listFiles().length != 0 || dir2.listFiles().length != 0) {
      Thread.sleep(100);
    }
    
    // then increase the file's replication factor
    fs.setReplication(fileName, (short)2);
    // replication should succeed
    DFSTestUtil.waitReplication(fs, fileName, (short)1);
    
    // clean up the file
    fs.delete(fileName, false);
  } finally {
    cluster.shutdown();
  }
}
 

@Test
public void testTransferRbw() throws Exception {
  final HdfsConfiguration conf = new HdfsConfiguration();
  final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf
      ).numDataNodes(REPLICATION).build();
  try {
    cluster.waitActive();
    final DistributedFileSystem fs = cluster.getFileSystem();

    //create a file, write some data and leave it open. 
    final Path p = new Path("/foo");
    final int size = (1 << 16) + RAN.nextInt(1 << 16);
    LOG.info("size = " + size);
    final FSDataOutputStream out = fs.create(p, REPLICATION);
    final byte[] bytes = new byte[1024];
    for(int remaining = size; remaining > 0; ) {
      RAN.nextBytes(bytes);
      final int len = bytes.length < remaining? bytes.length: remaining;
      out.write(bytes, 0, len);
      out.hflush();
      remaining -= len;
    }

    //get the RBW
    final ReplicaBeingWritten oldrbw;
    final DataNode newnode;
    final DatanodeInfo newnodeinfo;
    final String bpid = cluster.getNamesystem().getBlockPoolId();
    {
      final DataNode oldnode = cluster.getDataNodes().get(0);
      oldrbw = getRbw(oldnode, bpid);
      LOG.info("oldrbw = " + oldrbw);
      
      //add a datanode
      cluster.startDataNodes(conf, 1, true, null, null);
      newnode = cluster.getDataNodes().get(REPLICATION);
      
      final DatanodeInfo oldnodeinfo;
      {
        final DatanodeInfo[] datatnodeinfos = cluster.getNameNodeRpc(
            ).getDatanodeReport(DatanodeReportType.LIVE);
        Assert.assertEquals(2, datatnodeinfos.length);
        int i = 0;
        for(DatanodeRegistration dnReg = newnode.getDNRegistrationForBP(bpid);
            i < datatnodeinfos.length && !datatnodeinfos[i].equals(dnReg); i++);
        Assert.assertTrue(i < datatnodeinfos.length);
        newnodeinfo = datatnodeinfos[i];
        oldnodeinfo = datatnodeinfos[1 - i];
      }
      
      //transfer RBW
      final ExtendedBlock b = new ExtendedBlock(bpid, oldrbw.getBlockId(), oldrbw.getBytesAcked(),
          oldrbw.getGenerationStamp());
      final BlockOpResponseProto s = DFSTestUtil.transferRbw(
          b, DFSClientAdapter.getDFSClient(fs), oldnodeinfo, newnodeinfo);
      Assert.assertEquals(Status.SUCCESS, s.getStatus());
    }

    //check new rbw
    final ReplicaBeingWritten newrbw = getRbw(newnode, bpid);
    LOG.info("newrbw = " + newrbw);
    Assert.assertEquals(oldrbw.getBlockId(), newrbw.getBlockId());
    Assert.assertEquals(oldrbw.getGenerationStamp(), newrbw.getGenerationStamp());
    Assert.assertEquals(oldrbw.getVisibleLength(), newrbw.getVisibleLength());

    LOG.info("DONE");
  } finally {
    cluster.shutdown();
  }
}
 

/**
 * The test verifies that replica for deletion is chosen on a node,
 * with the oldest heartbeat, when this heartbeat is larger than the
 * tolerable heartbeat interval.
 * It creates a file with several blocks and replication 4.
 * The last DN is configured to send heartbeats rarely.
 * 
 * Test waits until the tolerable heartbeat interval expires, and reduces
 * replication of the file. All replica deletions should be scheduled for the
 * last node. No replicas will actually be deleted, since last DN doesn't
 * send heartbeats. 
 */
@Test
public void testChooseReplicaToDelete() throws Exception {
  MiniDFSCluster cluster = null;
  FileSystem fs = null;
  try {
    Configuration conf = new HdfsConfiguration();
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, SMALL_BLOCK_SIZE);
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    fs = cluster.getFileSystem();
    final FSNamesystem namesystem = cluster.getNamesystem();

    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 300);
    cluster.startDataNodes(conf, 1, true, null, null, null);
    DataNode lastDN = cluster.getDataNodes().get(3);
    DatanodeRegistration dnReg = DataNodeTestUtils.getDNRegistrationForBP(
        lastDN, namesystem.getBlockPoolId());
    String lastDNid = dnReg.getDatanodeUuid();

    final Path fileName = new Path("/foo2");
    DFSTestUtil.createFile(fs, fileName, SMALL_FILE_LENGTH, (short)4, 0L);
    DFSTestUtil.waitReplication(fs, fileName, (short)4);

    // Wait for tolerable number of heartbeats plus one
    DatanodeDescriptor nodeInfo = null;
    long lastHeartbeat = 0;
    long waitTime = DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT * 1000 *
      (DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_DEFAULT + 1);
    do {
      nodeInfo = namesystem.getBlockManager().getDatanodeManager()
          .getDatanode(dnReg);
      lastHeartbeat = nodeInfo.getLastUpdateMonotonic();
    } while (monotonicNow() - lastHeartbeat < waitTime);
    fs.setReplication(fileName, (short)3);

    BlockLocation locs[] = fs.getFileBlockLocations(
        fs.getFileStatus(fileName), 0, Long.MAX_VALUE);

    // All replicas for deletion should be scheduled on lastDN.
    // And should not actually be deleted, because lastDN does not heartbeat.
    namesystem.readLock();
    Collection<Block> dnBlocks = 
      namesystem.getBlockManager().excessReplicateMap.get(lastDNid);
    assertEquals("Replicas on node " + lastDNid + " should have been deleted",
        SMALL_FILE_LENGTH / SMALL_BLOCK_SIZE, dnBlocks.size());
    namesystem.readUnlock();
    for(BlockLocation location : locs)
      assertEquals("Block should still have 4 replicas",
          4, location.getNames().length);
  } finally {
    if(fs != null) fs.close();
    if(cluster != null) cluster.shutdown();
  }
}
 

@Test
public void testReduceReplFactorDueToRejoinRespectsRackPolicy() 
    throws Exception {
  Configuration conf = getConf();
  short REPLICATION_FACTOR = 2;
  final Path filePath = new Path("/testFile");
  // Last datanode is on a different rack
  String racks[] = {"/rack1", "/rack1", "/rack2"};
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
    .numDataNodes(racks.length).racks(racks).build();
  final FSNamesystem ns = cluster.getNameNode().getNamesystem();
  final DatanodeManager dm = ns.getBlockManager().getDatanodeManager();

  try {
    // Create a file with one block
    final FileSystem fs = cluster.getFileSystem();
    DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
    ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
    DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);

    // Make the last (cross rack) datanode look like it failed
    // to heartbeat by stopping it and calling removeDatanode.
    ArrayList<DataNode> datanodes = cluster.getDataNodes();
    assertEquals(3, datanodes.size());
    DataNode dataNode = datanodes.get(2);
    DatanodeID dnId = dataNode.getDatanodeId();
    cluster.stopDataNode(2);
    dm.removeDatanode(dnId);

    // The block gets re-replicated to another datanode so it has a 
    // sufficient # replicas, but not across racks, so there should
    // be 1 rack, and 1 needed replica (even though there are 2 hosts 
    // available and only 2 replicas required).
    DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 1);

    // Start the "failed" datanode, which has a replica so the block is
    // now over-replicated and therefore a replica should be removed but
    // not on the restarted datanode as that would violate the rack policy.
    String rack2[] = {"/rack2"};
    cluster.startDataNodes(conf, 1, true, null, rack2);
    cluster.waitActive();      
    
    // The block now has sufficient # replicas, across racks
    DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
  } finally {
    cluster.shutdown();
  }
}
 

@Test
public void testNodeCount() throws Exception {
  // start a mini dfs cluster of 2 nodes
  final Configuration conf = new HdfsConfiguration();
  final MiniDFSCluster cluster = 
    new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION_FACTOR).build();
  try {
    final FSNamesystem namesystem = cluster.getNamesystem();
    final BlockManager bm = namesystem.getBlockManager();
    final HeartbeatManager hm = bm.getDatanodeManager().getHeartbeatManager();
    final FileSystem fs = cluster.getFileSystem();
    
    // populate the cluster with a one block file
    final Path FILE_PATH = new Path("/testfile");
    DFSTestUtil.createFile(fs, FILE_PATH, 1L, REPLICATION_FACTOR, 1L);
    DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR);
    ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, FILE_PATH);

    // keep a copy of all datanode descriptor
    final DatanodeDescriptor[] datanodes = hm.getDatanodes();
    
    // start two new nodes
    cluster.startDataNodes(conf, 2, true, null, null);
    cluster.waitActive();
    
    // bring down first datanode
    DatanodeDescriptor datanode = datanodes[0];
    DataNodeProperties dnprop = cluster.stopDataNode(datanode.getXferAddr());
    
    // make sure that NN detects that the datanode is down
    BlockManagerTestUtil.noticeDeadDatanode(
        cluster.getNameNode(), datanode.getXferAddr());
    
    // the block will be replicated
    DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR);

    // restart the first datanode
    cluster.restartDataNode(dnprop);
    cluster.waitActive();
    
    // check if excessive replica is detected (transient)
    initializeTimeout(TIMEOUT);
    while (countNodes(block.getLocalBlock(), namesystem).excessReplicas() == 0) {
      checkTimeout("excess replicas not detected");
    }
    
    // find out a non-excess node
    DatanodeDescriptor nonExcessDN = null;
    for(DatanodeStorageInfo storage : bm.blocksMap.getStorages(block.getLocalBlock())) {
      final DatanodeDescriptor dn = storage.getDatanodeDescriptor();
      Collection<Block> blocks = bm.excessReplicateMap.get(dn.getDatanodeUuid());
      if (blocks == null || !blocks.contains(block.getLocalBlock()) ) {
        nonExcessDN = dn;
        break;
      }
    }
    assertTrue(nonExcessDN!=null);
    
    // bring down non excessive datanode
    dnprop = cluster.stopDataNode(nonExcessDN.getXferAddr());
    // make sure that NN detects that the datanode is down
    BlockManagerTestUtil.noticeDeadDatanode(
        cluster.getNameNode(), nonExcessDN.getXferAddr());

    // The block should be replicated
    initializeTimeout(TIMEOUT);
    while (countNodes(block.getLocalBlock(), namesystem).liveReplicas() != REPLICATION_FACTOR) {
      checkTimeout("live replica count not correct", 1000);
    }

    // restart the first datanode
    cluster.restartDataNode(dnprop);
    cluster.waitActive();

    // check if excessive replica is detected (transient)
    initializeTimeout(TIMEOUT);
    while (countNodes(block.getLocalBlock(), namesystem).excessReplicas() != 2) {
      checkTimeout("excess replica count not equal to 2");
    }

  } finally {
    cluster.shutdown();
  }
}
 

/**
 * Test when a block's replica is removed from RBW folder in one of the
 * datanode, namenode should ask to invalidate that corrupted block and
 * schedule replication for one more replica for that under replicated block.
 */
@Test(timeout=600000)
public void testBlockInvalidationWhenRBWReplicaMissedInDN()
    throws IOException, InterruptedException {
  // This test cannot pass on Windows due to file locking enforcement.  It will
  // reject the attempt to delete the block file from the RBW folder.
  assumeTrue(!Path.WINDOWS);

  Configuration conf = new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2);
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 300);
  conf.setLong(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
  conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
      .build();
  FSDataOutputStream out = null;
  try {
    final FSNamesystem namesystem = cluster.getNamesystem();
    FileSystem fs = cluster.getFileSystem();
    Path testPath = new Path("/tmp/TestRBWBlockInvalidation", "foo1");
    out = fs.create(testPath, (short) 2);
    out.writeBytes("HDFS-3157: " + testPath);
    out.hsync();
    cluster.startDataNodes(conf, 1, true, null, null, null);
    String bpid = namesystem.getBlockPoolId();
    ExtendedBlock blk = DFSTestUtil.getFirstBlock(fs, testPath);
    Block block = blk.getLocalBlock();
    DataNode dn = cluster.getDataNodes().get(0);

    // Delete partial block and its meta information from the RBW folder
    // of first datanode.
    File blockFile = DataNodeTestUtils.getBlockFile(dn, bpid, block);
    File metaFile = DataNodeTestUtils.getMetaFile(dn, bpid, block);
    assertTrue("Could not delete the block file from the RBW folder",
        blockFile.delete());
    assertTrue("Could not delete the block meta file from the RBW folder",
        metaFile.delete());

    out.close();
    
    int liveReplicas = 0;
    while (true) {
      if ((liveReplicas = countReplicas(namesystem, blk).liveReplicas()) < 2) {
        // This confirms we have a corrupt replica
        LOG.info("Live Replicas after corruption: " + liveReplicas);
        break;
      }
      Thread.sleep(100);
    }
    assertEquals("There should be less than 2 replicas in the "
        + "liveReplicasMap", 1, liveReplicas);
    
    while (true) {
      if ((liveReplicas =
            countReplicas(namesystem, blk).liveReplicas()) > 1) {
        //Wait till the live replica count becomes equal to Replication Factor
        LOG.info("Live Replicas after Rereplication: " + liveReplicas);
        break;
      }
      Thread.sleep(100);
    }
    assertEquals("There should be two live replicas", 2, liveReplicas);

    while (true) {
      Thread.sleep(100);
      if (countReplicas(namesystem, blk).corruptReplicas() == 0) {
        LOG.info("Corrupt Replicas becomes 0");
        break;
      }
    }
  } finally {
    if (out != null) {
      out.close();
    }
    cluster.shutdown();
  }
}
 

/**
 * The test verifies the number of outstanding replication requests for a
 * given DN shouldn't exceed the limit set by configuration property
 * dfs.namenode.replication.max-streams-hard-limit.
 * The test does the followings:
 * 1. Create a mini cluster with 2 DNs. Set large heartbeat interval so that
 *    replication requests won't be picked by any DN right away.
 * 2. Create a file with 10 blocks and replication factor 2. Thus each
 *    of the 2 DNs have one replica of each block.
 * 3. Add a DN to the cluster for later replication.
 * 4. Remove a DN that has data.
 * 5. Ask BlockManager to compute the replication work. This will assign
 *    replication requests to the only DN that has data.
 * 6. Make sure the number of pending replication requests of that DN don't
 *    exceed the limit.
 * @throws Exception
 */
@Test(timeout=60000) // 1 min timeout
public void testNumberOfBlocksToBeReplicated() throws Exception {
  Configuration conf = new HdfsConfiguration();

  conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0);
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1);
  conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);

  // Large value to make sure the pending replication request can stay in
  // DatanodeDescriptor.replicateBlocks before test timeout.
  conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 100);

  // Make sure BlockManager can pull all blocks from UnderReplicatedBlocks via
  // chooseUnderReplicatedBlocks at once.
   conf.setInt(
      DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, 5);

  int NUM_OF_BLOCKS = 10;
  final short REP_FACTOR = 2;
  final String FILE_NAME = "/testFile";
  final Path FILE_PATH = new Path(FILE_NAME);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
          REP_FACTOR).build();
  try {
    // create a file with 10 blocks with a replication factor of 2
    final FileSystem fs = cluster.getFileSystem();
    DFSTestUtil.createFile(fs, FILE_PATH, NUM_OF_BLOCKS, REP_FACTOR, 1L);
    DFSTestUtil.waitReplication(fs, FILE_PATH, REP_FACTOR);

    cluster.startDataNodes(conf, 1, true, null, null, null, null);

    final BlockManager bm = cluster.getNamesystem().getBlockManager();
    ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, FILE_PATH);
    Iterator<DatanodeStorageInfo> storageInfos =
        bm.blocksMap.getStorages(b.getLocalBlock())
        .iterator();
    DatanodeDescriptor firstDn = storageInfos.next().getDatanodeDescriptor();
    DatanodeDescriptor secondDn = storageInfos.next().getDatanodeDescriptor();

    bm.getDatanodeManager().removeDatanode(firstDn);

    assertEquals(NUM_OF_BLOCKS, bm.getUnderReplicatedNotMissingBlocks());
    bm.computeDatanodeWork();


    assertTrue("The number of blocks to be replicated should be less than "
        + "or equal to " + bm.replicationStreamsHardLimit,
        secondDn.getNumberOfBlocksToBeReplicated()
        <= bm.replicationStreamsHardLimit);
  } finally {
    cluster.shutdown();
  }

}
 

public void testReplicationError() throws Exception {
  // bring up a cluster of 1
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
  cluster.waitActive();
  FileSystem fs = cluster.getFileSystem();

  try {
    // create a file of replication factor of 1
    final Path fileName = new Path("/test.txt");
    final int fileLen = 1;
    DFSTestUtil.createFile(fs, fileName, 1, (short)1, 1L);
    DFSTestUtil.waitReplication(fs, fileName, (short)1);

    // get the block belonged to the created file
    LocatedBlocks blocks = cluster.getNameNode().namesystem.getBlockLocations(
        fileName.toString(), 0, (long)fileLen);
    assertEquals(blocks.locatedBlockCount(), 1);
    LocatedBlock block = blocks.get(0);

    // bring up a second datanode
    cluster.startDataNodes(conf, 1, true, null, null);
    cluster.waitActive();
    final int sndNode = 1;
    DataNode datanode = cluster.getDataNodes().get(sndNode);

    // replicate the block to the second datanode
    InetSocketAddress target = datanode.getSelfAddr();
    Socket s = new Socket(target.getAddress(), target.getPort());
      //write the header.
    DataOutputStream out = new DataOutputStream(
        s.getOutputStream());

    out.writeShort( DataTransferProtocol.DATA_TRANSFER_VERSION );
    out.write( DataTransferProtocol.OP_WRITE_BLOCK );
    out.writeLong( block.getBlock().getBlockId());
    out.writeLong( block.getBlock().getGenerationStamp() );
    out.writeInt(1);
    out.writeBoolean( false );       // recovery flag
    Text.writeString( out, "" );
    out.writeBoolean(false); // Not sending src node information
    out.writeInt(0);
    
    // write check header
    out.writeByte( 1 );
    out.writeInt( 512 );

    out.flush();

    // close the connection before sending the content of the block
    out.close();
    
    // the temporary block & meta files should be deleted
    File dir1 = new File(cluster.getBlockDirectory(
          "data"+(2*sndNode+1)).getParentFile().getParent(), "tmp");
    File dir2 = new File(cluster.getBlockDirectory(
          "data"+(2*sndNode+2)).getParentFile().getParent(), "tmp");
    while (dir1.listFiles().length != 0 || dir2.listFiles().length != 0) {
      Thread.sleep(100);
    }
    
    // then increase the file's replication factor
    fs.setReplication(fileName, (short)2);
    // replication should succeed
    DFSTestUtil.waitReplication(fs, fileName, (short)1);
    
    // clean up the file
    fs.delete(fileName, false);
  } finally {
    cluster.shutdown();
  }
}