Java Code Examples for org.apache.hadoop.hdfs.protocol.ExtendedBlock#getNumBytes()

private static ExtendedBlock[][] generateBlocks(Suite s, long size
    ) throws IOException, InterruptedException, TimeoutException {
  final ExtendedBlock[][] blocks = new ExtendedBlock[s.clients.length][];
  for(int n = 0; n < s.clients.length; n++) {
    final long fileLen = size/s.replication;
    createFile(s, n, fileLen);

    final List<LocatedBlock> locatedBlocks = s.clients[n].getBlockLocations(
        FILE_NAME, 0, fileLen).getLocatedBlocks();

    final int numOfBlocks = locatedBlocks.size();
    blocks[n] = new ExtendedBlock[numOfBlocks];
    for(int i = 0; i < numOfBlocks; i++) {
      final ExtendedBlock b = locatedBlocks.get(i).getBlock();
      blocks[n][i] = new ExtendedBlock(b.getBlockPoolId(), b.getBlockId(),
          b.getNumBytes(), b.getGenerationStamp());
    }
  }
  return blocks;
}
 

private void writeZeroLengthPacket(ExtendedBlock block, String description)
throws IOException {
  PacketHeader hdr = new PacketHeader(
    8,                   // size of packet
    block.getNumBytes(), // OffsetInBlock
    100,                 // sequencenumber
    true,                // lastPacketInBlock
    0,                   // chunk length
    false);               // sync block
  hdr.write(sendOut);
  sendOut.writeInt(0);           // zero checksum

  //ok finally write a block with 0 len
  sendResponse(Status.SUCCESS, "", null, recvOut);
  new PipelineAck(100, new int[] {PipelineAck.combineHeader
    (PipelineAck.ECN.DISABLED, Status.SUCCESS)}).write
    (recvOut);
  sendRecvData(description, false);
}
 

private static ExtendedBlock[][] generateBlocks(Suite s, long size
    ) throws IOException, InterruptedException, TimeoutException {
  final ExtendedBlock[][] blocks = new ExtendedBlock[s.clients.length][];
  for(int n = 0; n < s.clients.length; n++) {
    final long fileLen = size/s.replication;
    createFile(s, n, fileLen);

    final List<LocatedBlock> locatedBlocks = s.clients[n].getBlockLocations(
        FILE_NAME, 0, fileLen).getLocatedBlocks();

    final int numOfBlocks = locatedBlocks.size();
    blocks[n] = new ExtendedBlock[numOfBlocks];
    for(int i = 0; i < numOfBlocks; i++) {
      final ExtendedBlock b = locatedBlocks.get(i).getBlock();
      blocks[n][i] = new ExtendedBlock(b.getBlockPoolId(), b.getBlockId(),
          b.getNumBytes(), b.getGenerationStamp());
    }
  }
  return blocks;
}
 

private void writeZeroLengthPacket(ExtendedBlock block, String description)
throws IOException {
  PacketHeader hdr = new PacketHeader(
    8,                   // size of packet
    block.getNumBytes(), // OffsetInBlock
    100,                 // sequencenumber
    true,                // lastPacketInBlock
    0,                   // chunk length
    false);               // sync block
  hdr.write(sendOut);
  sendOut.writeInt(0);           // zero checksum

  //ok finally write a block with 0 len
  sendResponse(Status.SUCCESS, "", null, recvOut);
  new PipelineAck(100, new int[] {PipelineAck.combineHeader
    (PipelineAck.ECN.DISABLED, Status.SUCCESS)}).write
    (recvOut);
  sendRecvData(description, false);
}
 

private void copyBlock(final DFSClient dfs, LocatedBlock lblock,
                       OutputStream fos) throws Exception {
  int failures = 0;
  InetSocketAddress targetAddr = null;
  TreeSet<DatanodeInfo> deadNodes = new TreeSet<DatanodeInfo>();
  BlockReader blockReader = null; 
  ExtendedBlock block = lblock.getBlock(); 

  while (blockReader == null) {
    DatanodeInfo chosenNode;
    
    try {
      chosenNode = bestNode(dfs, lblock.getLocations(), deadNodes);
      targetAddr = NetUtils.createSocketAddr(chosenNode.getXferAddr());
    }  catch (IOException ie) {
      if (failures >= DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT) {
        throw new IOException("Could not obtain block " + lblock, ie);
      }
      LOG.info("Could not obtain block from any node:  " + ie);
      try {
        Thread.sleep(10000);
      }  catch (InterruptedException iex) {
      }
      deadNodes.clear();
      failures++;
      continue;
    }
    try {
      String file = BlockReaderFactory.getFileName(targetAddr,
          block.getBlockPoolId(), block.getBlockId());
      blockReader = new BlockReaderFactory(dfs.getConf()).
          setFileName(file).
          setBlock(block).
          setBlockToken(lblock.getBlockToken()).
          setStartOffset(0).
          setLength(-1).
          setVerifyChecksum(true).
          setClientName("fsck").
          setDatanodeInfo(chosenNode).
          setInetSocketAddress(targetAddr).
          setCachingStrategy(CachingStrategy.newDropBehind()).
          setClientCacheContext(dfs.getClientContext()).
          setConfiguration(namenode.conf).
          setRemotePeerFactory(new RemotePeerFactory() {
            @Override
            public Peer newConnectedPeer(InetSocketAddress addr,
                Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
                throws IOException {
              Peer peer = null;
              Socket s = NetUtils.getDefaultSocketFactory(conf).createSocket();
              try {
                s.connect(addr, HdfsServerConstants.READ_TIMEOUT);
                s.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
                peer = TcpPeerServer.peerFromSocketAndKey(
                      dfs.getSaslDataTransferClient(), s, NamenodeFsck.this,
                      blockToken, datanodeId);
              } finally {
                if (peer == null) {
                  IOUtils.closeQuietly(s);
                }
              }
              return peer;
            }
          }).
          build();
    }  catch (IOException ex) {
      // Put chosen node into dead list, continue
      LOG.info("Failed to connect to " + targetAddr + ":" + ex);
      deadNodes.add(chosenNode);
    }
  }
  byte[] buf = new byte[1024];
  int cnt = 0;
  boolean success = true;
  long bytesRead = 0;
  try {
    while ((cnt = blockReader.read(buf, 0, buf.length)) > 0) {
      fos.write(buf, 0, cnt);
      bytesRead += cnt;
    }
    if ( bytesRead != block.getNumBytes() ) {
      throw new IOException("Recorded block size is " + block.getNumBytes() + 
                            ", but datanode returned " +bytesRead+" bytes");
    }
  } catch (Exception e) {
    LOG.error("Error reading block", e);
    success = false;
  } finally {
    blockReader.close();
  }
  if (!success) {
    throw new Exception("Could not copy block data for " + lblock.getBlock());
  }
}
 

/** 
 * Test  for
 * {@link FsDatasetImpl#updateReplicaUnderRecovery(ExtendedBlock, long, long)} 
 * */
@Test
public void testUpdateReplicaUnderRecovery() throws IOException {
  MiniDFSCluster cluster = null;

  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    cluster.waitActive();
    String bpid = cluster.getNamesystem().getBlockPoolId();

    //create a file
    DistributedFileSystem dfs = cluster.getFileSystem();
    String filestr = "/foo";
    Path filepath = new Path(filestr);
    DFSTestUtil.createFile(dfs, filepath, 1024L, (short)3, 0L);

    //get block info
    final LocatedBlock locatedblock = getLastLocatedBlock(
        DFSClientAdapter.getDFSClient(dfs).getNamenode(), filestr);
    final DatanodeInfo[] datanodeinfo = locatedblock.getLocations();
    Assert.assertTrue(datanodeinfo.length > 0);

    //get DataNode and FSDataset objects
    final DataNode datanode = cluster.getDataNode(datanodeinfo[0].getIpcPort());
    Assert.assertTrue(datanode != null);

    //initReplicaRecovery
    final ExtendedBlock b = locatedblock.getBlock();
    final long recoveryid = b.getGenerationStamp() + 1;
    final long newlength = b.getNumBytes() - 1;
    final FsDatasetSpi<?> fsdataset = DataNodeTestUtils.getFSDataset(datanode);
    final ReplicaRecoveryInfo rri = fsdataset.initReplicaRecovery(
        new RecoveringBlock(b, null, recoveryid));

    //check replica
    final ReplicaInfo replica = FsDatasetTestUtil.fetchReplicaInfo(
        fsdataset, bpid, b.getBlockId());
    Assert.assertEquals(ReplicaState.RUR, replica.getState());

    //check meta data before update
    FsDatasetImpl.checkReplicaFiles(replica);

    //case "THIS IS NOT SUPPOSED TO HAPPEN"
    //with (block length) != (stored replica's on disk length). 
    {
      //create a block with same id and gs but different length.
      final ExtendedBlock tmp = new ExtendedBlock(b.getBlockPoolId(), rri
          .getBlockId(), rri.getNumBytes() - 1, rri.getGenerationStamp());
      try {
        //update should fail
        fsdataset.updateReplicaUnderRecovery(tmp, recoveryid,
            tmp.getBlockId(), newlength);
        Assert.fail();
      } catch(IOException ioe) {
        System.out.println("GOOD: getting " + ioe);
      }
    }

    //update
    final String storageID = fsdataset.updateReplicaUnderRecovery(
        new ExtendedBlock(b.getBlockPoolId(), rri), recoveryid,
        rri.getBlockId(), newlength);
    assertTrue(storageID != null);

  } finally {
    if (cluster != null) cluster.shutdown();
  }
}
 

/**
 * The following test first creates a file.
 * It verifies the block information from a datanode.
 * Then, it updates the block with new information and verifies again.
 * @param useDnHostname whether DNs should connect to other DNs by hostname
 */
private void checkBlockMetaDataInfo(boolean useDnHostname) throws Exception {
  MiniDFSCluster cluster = null;

  conf.setBoolean(DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, useDnHostname);
  if (useDnHostname) {
    // Since the mini cluster only listens on the loopback we have to
    // ensure the hostname used to access DNs maps to the loopback. We
    // do this by telling the DN to advertise localhost as its hostname
    // instead of the default hostname.
    conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "localhost");
  }

  try {
    cluster = new MiniDFSCluster.Builder(conf)
      .numDataNodes(3)
      .checkDataNodeHostConfig(true)
      .build();
    cluster.waitActive();

    //create a file
    DistributedFileSystem dfs = cluster.getFileSystem();
    String filestr = "/foo";
    Path filepath = new Path(filestr);
    DFSTestUtil.createFile(dfs, filepath, 1024L, (short)3, 0L);
    assertTrue(dfs.exists(filepath));

    //get block info
    LocatedBlock locatedblock = getLastLocatedBlock(
        DFSClientAdapter.getDFSClient(dfs).getNamenode(), filestr);
    DatanodeInfo[] datanodeinfo = locatedblock.getLocations();
    assertTrue(datanodeinfo.length > 0);

    //connect to a data node
    DataNode datanode = cluster.getDataNode(datanodeinfo[0].getIpcPort());
    InterDatanodeProtocol idp = DataNodeTestUtils.createInterDatanodeProtocolProxy(
        datanode, datanodeinfo[0], conf, useDnHostname);
    
    // Stop the block scanners.
    datanode.getBlockScanner().removeAllVolumeScanners();

    //verify BlockMetaDataInfo
    ExtendedBlock b = locatedblock.getBlock();
    InterDatanodeProtocol.LOG.info("b=" + b + ", " + b.getClass());
    checkMetaInfo(b, datanode);
    long recoveryId = b.getGenerationStamp() + 1;
    idp.initReplicaRecovery(
        new RecoveringBlock(b, locatedblock.getLocations(), recoveryId));

    //verify updateBlock
    ExtendedBlock newblock = new ExtendedBlock(b.getBlockPoolId(),
        b.getBlockId(), b.getNumBytes()/2, b.getGenerationStamp()+1);
    idp.updateReplicaUnderRecovery(b, recoveryId, b.getBlockId(),
        newblock.getNumBytes());
    checkMetaInfo(newblock, datanode);
    
    // Verify correct null response trying to init recovery for a missing block
    ExtendedBlock badBlock = new ExtendedBlock("fake-pool",
        b.getBlockId(), 0, 0);
    assertNull(idp.initReplicaRecovery(
        new RecoveringBlock(badBlock,
            locatedblock.getLocations(), recoveryId)));
  }
  finally {
    if (cluster != null) {cluster.shutdown();}
  }
}
 

@Override // FsDatasetSpi
public long getReplicaVisibleLength(ExtendedBlock block) {
  return block.getNumBytes();
}
 

@Override
public void blockChecksum(final ExtendedBlock block,
    final Token<BlockTokenIdentifier> blockToken) throws IOException {
  final DataOutputStream out = new DataOutputStream(
      getOutputStream());
  checkAccess(out, true, block, blockToken,
      Op.BLOCK_CHECKSUM, BlockTokenSecretManager.AccessMode.READ);
  // client side now can specify a range of the block for checksum
  long requestLength = block.getNumBytes();
  Preconditions.checkArgument(requestLength >= 0);
  long visibleLength = datanode.data.getReplicaVisibleLength(block);
  boolean partialBlk = requestLength < visibleLength;

  updateCurrentThreadName("Reading metadata for block " + block);
  final LengthInputStream metadataIn = datanode.data
      .getMetaDataInputStream(block);
  
  final DataInputStream checksumIn = new DataInputStream(
      new BufferedInputStream(metadataIn, HdfsConstants.IO_FILE_BUFFER_SIZE));
  updateCurrentThreadName("Getting checksum for block " + block);
  try {
    //read metadata file
    final BlockMetadataHeader header = BlockMetadataHeader
        .readHeader(checksumIn);
    final DataChecksum checksum = header.getChecksum();
    final int csize = checksum.getChecksumSize();
    final int bytesPerCRC = checksum.getBytesPerChecksum();
    final long crcPerBlock = csize <= 0 ? 0 : 
      (metadataIn.getLength() - BlockMetadataHeader.getHeaderSize()) / csize;

    final MD5Hash md5 = partialBlk && crcPerBlock > 0 ? 
        calcPartialBlockChecksum(block, requestLength, checksum, checksumIn)
          : MD5Hash.digest(checksumIn);
    if (LOG.isDebugEnabled()) {
      LOG.debug("block=" + block + ", bytesPerCRC=" + bytesPerCRC
          + ", crcPerBlock=" + crcPerBlock + ", md5=" + md5);
    }

    //write reply
    BlockOpResponseProto.newBuilder()
      .setStatus(SUCCESS)
      .setChecksumResponse(OpBlockChecksumResponseProto.newBuilder()             
        .setBytesPerCrc(bytesPerCRC)
        .setCrcPerBlock(crcPerBlock)
        .setMd5(ByteString.copyFrom(md5.getDigest()))
        .setCrcType(PBHelper.convert(checksum.getChecksumType())))
      .build()
      .writeDelimitedTo(out);
    out.flush();
  } catch (IOException ioe) {
    LOG.info("blockChecksum " + block + " received exception " + ioe);
    incrDatanodeNetworkErrors();
    throw ioe;
  } finally {
    IOUtils.closeStream(out);
    IOUtils.closeStream(checksumIn);
    IOUtils.closeStream(metadataIn);
  }

  //update metrics
  datanode.metrics.addBlockChecksumOp(elapsed());
}
 

@Override // FsDatasetSpi
public synchronized ReplicaInPipeline convertTemporaryToRbw(
    final ExtendedBlock b) throws IOException {
  final long blockId = b.getBlockId();
  final long expectedGs = b.getGenerationStamp();
  final long visible = b.getNumBytes();
  LOG.info("Convert " + b + " from Temporary to RBW, visible length="
      + visible);

  final ReplicaInPipeline temp;
  {
    // get replica
    final ReplicaInfo r = volumeMap.get(b.getBlockPoolId(), blockId);
    if (r == null) {
      throw new ReplicaNotFoundException(
          ReplicaNotFoundException.NON_EXISTENT_REPLICA + b);
    }
    // check the replica's state
    if (r.getState() != ReplicaState.TEMPORARY) {
      throw new ReplicaAlreadyExistsException(
          "r.getState() != ReplicaState.TEMPORARY, r=" + r);
    }
    temp = (ReplicaInPipeline)r;
  }
  // check generation stamp
  if (temp.getGenerationStamp() != expectedGs) {
    throw new ReplicaAlreadyExistsException(
        "temp.getGenerationStamp() != expectedGs = " + expectedGs
        + ", temp=" + temp);
  }

  // TODO: check writer?
  // set writer to the current thread
  // temp.setWriter(Thread.currentThread());

  // check length
  final long numBytes = temp.getNumBytes();
  if (numBytes < visible) {
    throw new IOException(numBytes + " = numBytes < visible = "
        + visible + ", temp=" + temp);
  }
  // check volume
  final FsVolumeImpl v = (FsVolumeImpl)temp.getVolume();
  if (v == null) {
    throw new IOException("r.getVolume() = null, temp="  + temp);
  }
  
  // move block files to the rbw directory
  BlockPoolSlice bpslice = v.getBlockPoolSlice(b.getBlockPoolId());
  final File dest = moveBlockFiles(b.getLocalBlock(), temp.getBlockFile(), 
      bpslice.getRbwDir());
  // create RBW
  final ReplicaBeingWritten rbw = new ReplicaBeingWritten(
      blockId, numBytes, expectedGs,
      v, dest.getParentFile(), Thread.currentThread(), 0);
  rbw.setBytesAcked(visible);
  // overwrite the RBW in the volume map
  volumeMap.add(b.getBlockPoolId(), rbw);
  return rbw;
}
 

@Override // FsDatasetSpi
public synchronized ReplicaHandler createRbw(
    StorageType storageType, ExtendedBlock b, boolean allowLazyPersist)
    throws IOException {
  ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(),
      b.getBlockId());
  if (replicaInfo != null) {
    throw new ReplicaAlreadyExistsException("Block " + b +
    " already exists in state " + replicaInfo.getState() +
    " and thus cannot be created.");
  }
  // create a new block
  FsVolumeReference ref;
  while (true) {
    try {
      if (allowLazyPersist) {
        // First try to place the block on a transient volume.
        ref = volumes.getNextTransientVolume(b.getNumBytes());
        datanode.getMetrics().incrRamDiskBlocksWrite();
      } else {
        ref = volumes.getNextVolume(storageType, b.getNumBytes());
      }
    } catch (DiskOutOfSpaceException de) {
      if (allowLazyPersist) {
        datanode.getMetrics().incrRamDiskBlocksWriteFallback();
        allowLazyPersist = false;
        continue;
      }
      throw de;
    }
    break;
  }
  FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
  // create an rbw file to hold block in the designated volume
  File f;
  try {
    f = v.createRbwFile(b.getBlockPoolId(), b.getLocalBlock());
  } catch (IOException e) {
    IOUtils.cleanup(null, ref);
    throw e;
  }

  ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten(b.getBlockId(), 
      b.getGenerationStamp(), v, f.getParentFile(), b.getNumBytes());
  volumeMap.add(b.getBlockPoolId(), newReplicaInfo);
  return new ReplicaHandler(newReplicaInfo, ref);
}
 

/**
 * Move block files from one storage to another storage.
 * @return Returns the Old replicaInfo
 * @throws IOException
 */
@Override
public ReplicaInfo moveBlockAcrossStorage(ExtendedBlock block,
    StorageType targetStorageType) throws IOException {
  ReplicaInfo replicaInfo = getReplicaInfo(block);
  if (replicaInfo.getState() != ReplicaState.FINALIZED) {
    throw new ReplicaNotFoundException(
        ReplicaNotFoundException.UNFINALIZED_REPLICA + block);
  }
  if (replicaInfo.getNumBytes() != block.getNumBytes()) {
    throw new IOException("Corrupted replica " + replicaInfo
        + " with a length of " + replicaInfo.getNumBytes()
        + " expected length is " + block.getNumBytes());
  }
  if (replicaInfo.getVolume().getStorageType() == targetStorageType) {
    throw new ReplicaAlreadyExistsException("Replica " + replicaInfo
        + " already exists on storage " + targetStorageType);
  }

  if (replicaInfo.isOnTransientStorage()) {
    // Block movement from RAM_DISK will be done by LazyPersist mechanism
    throw new IOException("Replica " + replicaInfo
        + " cannot be moved from storageType : "
        + replicaInfo.getVolume().getStorageType());
  }

  try (FsVolumeReference volumeRef = volumes.getNextVolume(
      targetStorageType, block.getNumBytes())) {
    File oldBlockFile = replicaInfo.getBlockFile();
    File oldMetaFile = replicaInfo.getMetaFile();
    FsVolumeImpl targetVolume = (FsVolumeImpl) volumeRef.getVolume();
    // Copy files to temp dir first
    File[] blockFiles = copyBlockFiles(block.getBlockId(),
        block.getGenerationStamp(), oldMetaFile, oldBlockFile,
        targetVolume.getTmpDir(block.getBlockPoolId()),
        replicaInfo.isOnTransientStorage());

    ReplicaInfo newReplicaInfo = new ReplicaInPipeline(
        replicaInfo.getBlockId(), replicaInfo.getGenerationStamp(),
        targetVolume, blockFiles[0].getParentFile(), 0);
    newReplicaInfo.setNumBytes(blockFiles[1].length());
    // Finalize the copied files
    newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo);

    removeOldReplica(replicaInfo, newReplicaInfo, oldBlockFile, oldMetaFile,
        oldBlockFile.length(), oldMetaFile.length(), block.getBlockPoolId());
  }

  // Replace the old block if any to reschedule the scanning.
  return replicaInfo;
}
 

private void copyBlock(final DFSClient dfs, LocatedBlock lblock,
                       OutputStream fos) throws Exception {
  int failures = 0;
  InetSocketAddress targetAddr = null;
  TreeSet<DatanodeInfo> deadNodes = new TreeSet<DatanodeInfo>();
  BlockReader blockReader = null; 
  ExtendedBlock block = lblock.getBlock(); 

  while (blockReader == null) {
    DatanodeInfo chosenNode;
    
    try {
      chosenNode = bestNode(dfs, lblock.getLocations(), deadNodes);
      targetAddr = NetUtils.createSocketAddr(chosenNode.getXferAddr());
    }  catch (IOException ie) {
      if (failures >= DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT) {
        throw new IOException("Could not obtain block " + lblock, ie);
      }
      LOG.info("Could not obtain block from any node:  " + ie);
      try {
        Thread.sleep(10000);
      }  catch (InterruptedException iex) {
      }
      deadNodes.clear();
      failures++;
      continue;
    }
    try {
      String file = BlockReaderFactory.getFileName(targetAddr,
          block.getBlockPoolId(), block.getBlockId());
      blockReader = new BlockReaderFactory(dfs.getConf()).
          setFileName(file).
          setBlock(block).
          setBlockToken(lblock.getBlockToken()).
          setStartOffset(0).
          setLength(-1).
          setVerifyChecksum(true).
          setClientName("fsck").
          setDatanodeInfo(chosenNode).
          setInetSocketAddress(targetAddr).
          setCachingStrategy(CachingStrategy.newDropBehind()).
          setClientCacheContext(dfs.getClientContext()).
          setConfiguration(namenode.conf).
          setRemotePeerFactory(new RemotePeerFactory() {
            @Override
            public Peer newConnectedPeer(InetSocketAddress addr,
                Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
                throws IOException {
              Peer peer = null;
              Socket s = NetUtils.getDefaultSocketFactory(conf).createSocket();
              try {
                s.connect(addr, HdfsServerConstants.READ_TIMEOUT);
                s.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
                peer = TcpPeerServer.peerFromSocketAndKey(
                      dfs.getSaslDataTransferClient(), s, NamenodeFsck.this,
                      blockToken, datanodeId);
              } finally {
                if (peer == null) {
                  IOUtils.closeQuietly(s);
                }
              }
              return peer;
            }
          }).
          build();
    }  catch (IOException ex) {
      // Put chosen node into dead list, continue
      LOG.info("Failed to connect to " + targetAddr + ":" + ex);
      deadNodes.add(chosenNode);
    }
  }
  byte[] buf = new byte[1024];
  int cnt = 0;
  boolean success = true;
  long bytesRead = 0;
  try {
    while ((cnt = blockReader.read(buf, 0, buf.length)) > 0) {
      fos.write(buf, 0, cnt);
      bytesRead += cnt;
    }
    if ( bytesRead != block.getNumBytes() ) {
      throw new IOException("Recorded block size is " + block.getNumBytes() + 
                            ", but datanode returned " +bytesRead+" bytes");
    }
  } catch (Exception e) {
    LOG.error("Error reading block", e);
    success = false;
  } finally {
    blockReader.close();
  }
  if (!success) {
    throw new Exception("Could not copy block data for " + lblock.getBlock());
  }
}
 

/** 
 * Test  for
 * {@link FsDatasetImpl#updateReplicaUnderRecovery(ExtendedBlock, long, long)} 
 * */
@Test
public void testUpdateReplicaUnderRecovery() throws IOException {
  MiniDFSCluster cluster = null;

  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    cluster.waitActive();
    String bpid = cluster.getNamesystem().getBlockPoolId();

    //create a file
    DistributedFileSystem dfs = cluster.getFileSystem();
    String filestr = "/foo";
    Path filepath = new Path(filestr);
    DFSTestUtil.createFile(dfs, filepath, 1024L, (short)3, 0L);

    //get block info
    final LocatedBlock locatedblock = getLastLocatedBlock(
        DFSClientAdapter.getDFSClient(dfs).getNamenode(), filestr);
    final DatanodeInfo[] datanodeinfo = locatedblock.getLocations();
    Assert.assertTrue(datanodeinfo.length > 0);

    //get DataNode and FSDataset objects
    final DataNode datanode = cluster.getDataNode(datanodeinfo[0].getIpcPort());
    Assert.assertTrue(datanode != null);

    //initReplicaRecovery
    final ExtendedBlock b = locatedblock.getBlock();
    final long recoveryid = b.getGenerationStamp() + 1;
    final long newlength = b.getNumBytes() - 1;
    final FsDatasetSpi<?> fsdataset = DataNodeTestUtils.getFSDataset(datanode);
    final ReplicaRecoveryInfo rri = fsdataset.initReplicaRecovery(
        new RecoveringBlock(b, null, recoveryid));

    //check replica
    final ReplicaInfo replica = FsDatasetTestUtil.fetchReplicaInfo(
        fsdataset, bpid, b.getBlockId());
    Assert.assertEquals(ReplicaState.RUR, replica.getState());

    //check meta data before update
    FsDatasetImpl.checkReplicaFiles(replica);

    //case "THIS IS NOT SUPPOSED TO HAPPEN"
    //with (block length) != (stored replica's on disk length). 
    {
      //create a block with same id and gs but different length.
      final ExtendedBlock tmp = new ExtendedBlock(b.getBlockPoolId(), rri
          .getBlockId(), rri.getNumBytes() - 1, rri.getGenerationStamp());
      try {
        //update should fail
        fsdataset.updateReplicaUnderRecovery(tmp, recoveryid,
            tmp.getBlockId(), newlength);
        Assert.fail();
      } catch(IOException ioe) {
        System.out.println("GOOD: getting " + ioe);
      }
    }

    //update
    final String storageID = fsdataset.updateReplicaUnderRecovery(
        new ExtendedBlock(b.getBlockPoolId(), rri), recoveryid,
        rri.getBlockId(), newlength);
    assertTrue(storageID != null);

  } finally {
    if (cluster != null) cluster.shutdown();
  }
}
 

/**
 * The following test first creates a file.
 * It verifies the block information from a datanode.
 * Then, it updates the block with new information and verifies again.
 * @param useDnHostname whether DNs should connect to other DNs by hostname
 */
private void checkBlockMetaDataInfo(boolean useDnHostname) throws Exception {
  MiniDFSCluster cluster = null;

  conf.setBoolean(DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, useDnHostname);
  if (useDnHostname) {
    // Since the mini cluster only listens on the loopback we have to
    // ensure the hostname used to access DNs maps to the loopback. We
    // do this by telling the DN to advertise localhost as its hostname
    // instead of the default hostname.
    conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "localhost");
  }

  try {
    cluster = new MiniDFSCluster.Builder(conf)
      .numDataNodes(3)
      .checkDataNodeHostConfig(true)
      .build();
    cluster.waitActive();

    //create a file
    DistributedFileSystem dfs = cluster.getFileSystem();
    String filestr = "/foo";
    Path filepath = new Path(filestr);
    DFSTestUtil.createFile(dfs, filepath, 1024L, (short)3, 0L);
    assertTrue(dfs.exists(filepath));

    //get block info
    LocatedBlock locatedblock = getLastLocatedBlock(
        DFSClientAdapter.getDFSClient(dfs).getNamenode(), filestr);
    DatanodeInfo[] datanodeinfo = locatedblock.getLocations();
    assertTrue(datanodeinfo.length > 0);

    //connect to a data node
    DataNode datanode = cluster.getDataNode(datanodeinfo[0].getIpcPort());
    InterDatanodeProtocol idp = DataNodeTestUtils.createInterDatanodeProtocolProxy(
        datanode, datanodeinfo[0], conf, useDnHostname);
    
    // Stop the block scanners.
    datanode.getBlockScanner().removeAllVolumeScanners();

    //verify BlockMetaDataInfo
    ExtendedBlock b = locatedblock.getBlock();
    InterDatanodeProtocol.LOG.info("b=" + b + ", " + b.getClass());
    checkMetaInfo(b, datanode);
    long recoveryId = b.getGenerationStamp() + 1;
    idp.initReplicaRecovery(
        new RecoveringBlock(b, locatedblock.getLocations(), recoveryId));

    //verify updateBlock
    ExtendedBlock newblock = new ExtendedBlock(b.getBlockPoolId(),
        b.getBlockId(), b.getNumBytes()/2, b.getGenerationStamp()+1);
    idp.updateReplicaUnderRecovery(b, recoveryId, b.getBlockId(),
        newblock.getNumBytes());
    checkMetaInfo(newblock, datanode);
    
    // Verify correct null response trying to init recovery for a missing block
    ExtendedBlock badBlock = new ExtendedBlock("fake-pool",
        b.getBlockId(), 0, 0);
    assertNull(idp.initReplicaRecovery(
        new RecoveringBlock(badBlock,
            locatedblock.getLocations(), recoveryId)));
  }
  finally {
    if (cluster != null) {cluster.shutdown();}
  }
}
 

@Override // FsDatasetSpi
public long getReplicaVisibleLength(ExtendedBlock block) {
  return block.getNumBytes();
}
 

@Override
public void blockChecksum(final ExtendedBlock block,
    final Token<BlockTokenIdentifier> blockToken) throws IOException {
  final DataOutputStream out = new DataOutputStream(
      getOutputStream());
  checkAccess(out, true, block, blockToken,
      Op.BLOCK_CHECKSUM, BlockTokenSecretManager.AccessMode.READ);
  // client side now can specify a range of the block for checksum
  long requestLength = block.getNumBytes();
  Preconditions.checkArgument(requestLength >= 0);
  long visibleLength = datanode.data.getReplicaVisibleLength(block);
  boolean partialBlk = requestLength < visibleLength;

  updateCurrentThreadName("Reading metadata for block " + block);
  final LengthInputStream metadataIn = datanode.data
      .getMetaDataInputStream(block);
  
  final DataInputStream checksumIn = new DataInputStream(
      new BufferedInputStream(metadataIn, HdfsConstants.IO_FILE_BUFFER_SIZE));
  updateCurrentThreadName("Getting checksum for block " + block);
  try {
    //read metadata file
    final BlockMetadataHeader header = BlockMetadataHeader
        .readHeader(checksumIn);
    final DataChecksum checksum = header.getChecksum();
    final int csize = checksum.getChecksumSize();
    final int bytesPerCRC = checksum.getBytesPerChecksum();
    final long crcPerBlock = csize <= 0 ? 0 : 
      (metadataIn.getLength() - BlockMetadataHeader.getHeaderSize()) / csize;

    final MD5Hash md5 = partialBlk && crcPerBlock > 0 ? 
        calcPartialBlockChecksum(block, requestLength, checksum, checksumIn)
          : MD5Hash.digest(checksumIn);
    if (LOG.isDebugEnabled()) {
      LOG.debug("block=" + block + ", bytesPerCRC=" + bytesPerCRC
          + ", crcPerBlock=" + crcPerBlock + ", md5=" + md5);
    }

    //write reply
    BlockOpResponseProto.newBuilder()
      .setStatus(SUCCESS)
      .setChecksumResponse(OpBlockChecksumResponseProto.newBuilder()             
        .setBytesPerCrc(bytesPerCRC)
        .setCrcPerBlock(crcPerBlock)
        .setMd5(ByteString.copyFrom(md5.getDigest()))
        .setCrcType(PBHelper.convert(checksum.getChecksumType())))
      .build()
      .writeDelimitedTo(out);
    out.flush();
  } catch (IOException ioe) {
    LOG.info("blockChecksum " + block + " received exception " + ioe);
    incrDatanodeNetworkErrors();
    throw ioe;
  } finally {
    IOUtils.closeStream(out);
    IOUtils.closeStream(checksumIn);
    IOUtils.closeStream(metadataIn);
  }

  //update metrics
  datanode.metrics.addBlockChecksumOp(elapsed());
}
 

@Override // FsDatasetSpi
public synchronized ReplicaInPipeline convertTemporaryToRbw(
    final ExtendedBlock b) throws IOException {
  final long blockId = b.getBlockId();
  final long expectedGs = b.getGenerationStamp();
  final long visible = b.getNumBytes();
  LOG.info("Convert " + b + " from Temporary to RBW, visible length="
      + visible);

  final ReplicaInPipeline temp;
  {
    // get replica
    final ReplicaInfo r = volumeMap.get(b.getBlockPoolId(), blockId);
    if (r == null) {
      throw new ReplicaNotFoundException(
          ReplicaNotFoundException.NON_EXISTENT_REPLICA + b);
    }
    // check the replica's state
    if (r.getState() != ReplicaState.TEMPORARY) {
      throw new ReplicaAlreadyExistsException(
          "r.getState() != ReplicaState.TEMPORARY, r=" + r);
    }
    temp = (ReplicaInPipeline)r;
  }
  // check generation stamp
  if (temp.getGenerationStamp() != expectedGs) {
    throw new ReplicaAlreadyExistsException(
        "temp.getGenerationStamp() != expectedGs = " + expectedGs
        + ", temp=" + temp);
  }

  // TODO: check writer?
  // set writer to the current thread
  // temp.setWriter(Thread.currentThread());

  // check length
  final long numBytes = temp.getNumBytes();
  if (numBytes < visible) {
    throw new IOException(numBytes + " = numBytes < visible = "
        + visible + ", temp=" + temp);
  }
  // check volume
  final FsVolumeImpl v = (FsVolumeImpl)temp.getVolume();
  if (v == null) {
    throw new IOException("r.getVolume() = null, temp="  + temp);
  }
  
  // move block files to the rbw directory
  BlockPoolSlice bpslice = v.getBlockPoolSlice(b.getBlockPoolId());
  final File dest = moveBlockFiles(b.getLocalBlock(), temp.getBlockFile(), 
      bpslice.getRbwDir());
  // create RBW
  final ReplicaBeingWritten rbw = new ReplicaBeingWritten(
      blockId, numBytes, expectedGs,
      v, dest.getParentFile(), Thread.currentThread(), 0);
  rbw.setBytesAcked(visible);
  // overwrite the RBW in the volume map
  volumeMap.add(b.getBlockPoolId(), rbw);
  return rbw;
}
 

@Override // FsDatasetSpi
public synchronized ReplicaHandler createRbw(
    StorageType storageType, ExtendedBlock b, boolean allowLazyPersist)
    throws IOException {
  ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(),
      b.getBlockId());
  if (replicaInfo != null) {
    throw new ReplicaAlreadyExistsException("Block " + b +
    " already exists in state " + replicaInfo.getState() +
    " and thus cannot be created.");
  }
  // create a new block
  FsVolumeReference ref;
  while (true) {
    try {
      if (allowLazyPersist) {
        // First try to place the block on a transient volume.
        ref = volumes.getNextTransientVolume(b.getNumBytes());
        datanode.getMetrics().incrRamDiskBlocksWrite();
      } else {
        ref = volumes.getNextVolume(storageType, b.getNumBytes());
      }
    } catch (DiskOutOfSpaceException de) {
      if (allowLazyPersist) {
        datanode.getMetrics().incrRamDiskBlocksWriteFallback();
        allowLazyPersist = false;
        continue;
      }
      throw de;
    }
    break;
  }
  FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
  // create an rbw file to hold block in the designated volume
  File f;
  try {
    f = v.createRbwFile(b.getBlockPoolId(), b.getLocalBlock());
  } catch (IOException e) {
    IOUtils.cleanup(null, ref);
    throw e;
  }

  ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten(b.getBlockId(), 
      b.getGenerationStamp(), v, f.getParentFile(), b.getNumBytes());
  volumeMap.add(b.getBlockPoolId(), newReplicaInfo);
  return new ReplicaHandler(newReplicaInfo, ref);
}
 

/**
 * Move block files from one storage to another storage.
 * @return Returns the Old replicaInfo
 * @throws IOException
 */
@Override
public ReplicaInfo moveBlockAcrossStorage(ExtendedBlock block,
    StorageType targetStorageType) throws IOException {
  ReplicaInfo replicaInfo = getReplicaInfo(block);
  if (replicaInfo.getState() != ReplicaState.FINALIZED) {
    throw new ReplicaNotFoundException(
        ReplicaNotFoundException.UNFINALIZED_REPLICA + block);
  }
  if (replicaInfo.getNumBytes() != block.getNumBytes()) {
    throw new IOException("Corrupted replica " + replicaInfo
        + " with a length of " + replicaInfo.getNumBytes()
        + " expected length is " + block.getNumBytes());
  }
  if (replicaInfo.getVolume().getStorageType() == targetStorageType) {
    throw new ReplicaAlreadyExistsException("Replica " + replicaInfo
        + " already exists on storage " + targetStorageType);
  }

  if (replicaInfo.isOnTransientStorage()) {
    // Block movement from RAM_DISK will be done by LazyPersist mechanism
    throw new IOException("Replica " + replicaInfo
        + " cannot be moved from storageType : "
        + replicaInfo.getVolume().getStorageType());
  }

  try (FsVolumeReference volumeRef = volumes.getNextVolume(
      targetStorageType, block.getNumBytes())) {
    File oldBlockFile = replicaInfo.getBlockFile();
    File oldMetaFile = replicaInfo.getMetaFile();
    FsVolumeImpl targetVolume = (FsVolumeImpl) volumeRef.getVolume();
    // Copy files to temp dir first
    File[] blockFiles = copyBlockFiles(block.getBlockId(),
        block.getGenerationStamp(), oldMetaFile, oldBlockFile,
        targetVolume.getTmpDir(block.getBlockPoolId()),
        replicaInfo.isOnTransientStorage());

    ReplicaInfo newReplicaInfo = new ReplicaInPipeline(
        replicaInfo.getBlockId(), replicaInfo.getGenerationStamp(),
        targetVolume, blockFiles[0].getParentFile(), 0);
    newReplicaInfo.setNumBytes(blockFiles[1].length());
    // Finalize the copied files
    newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo);

    removeOldReplica(replicaInfo, newReplicaInfo, oldBlockFile, oldMetaFile,
        oldBlockFile.length(), oldMetaFile.length(), block.getBlockPoolId());
  }

  // Replace the old block if any to reschedule the scanning.
  return replicaInfo;
}