Java Code Examples for org.apache.hadoop.hdfs.protocol.Block#setNumBytes()

BInfo(String bpid, Block b, boolean forWriting) throws IOException {
  theBlock = new Block(b);
  if (theBlock.getNumBytes() < 0) {
    theBlock.setNumBytes(0);
  }
  if (!storage.alloc(bpid, theBlock.getNumBytes())) { 
    // expected length - actual length may
    // be more - we find out at finalize
    DataNode.LOG.warn("Lack of free storage on a block alloc");
    throw new IOException("Creating block, no free space available");
  }

  if (forWriting) {
    finalized = false;
    oStream = new SimulatedOutputStream();
  } else {
    finalized = true;
    oStream = null;
  }
}
 

BInfo(String bpid, Block b, boolean forWriting) throws IOException {
  theBlock = new Block(b);
  if (theBlock.getNumBytes() < 0) {
    theBlock.setNumBytes(0);
  }
  if (!storage.alloc(bpid, theBlock.getNumBytes())) { 
    // expected length - actual length may
    // be more - we find out at finalize
    DataNode.LOG.warn("Lack of free storage on a block alloc");
    throw new IOException("Creating block, no free space available");
  }

  if (forWriting) {
    finalized = false;
    oStream = new SimulatedOutputStream();
  } else {
    finalized = true;
    oStream = null;
  }
}
 

BInfo(int namespaceId, Block b, boolean forWriting) throws IOException {
  theBlock = new Block(b);
  if (theBlock.getNumBytes() < 0) {
    theBlock.setNumBytes(0);
  }
  if (!storage.alloc(namespaceId, theBlock.getNumBytes())) { // expected length - actual length may
                                      // be more - we find out at finalize
    DataNode.LOG.warn("Lack of free storage on a block alloc");
    throw new IOException("Creating block, no free space available");
  }

  if (forWriting) {
    finalized = false;
    oStream = new SimulatedOutputStream();
  } else {
    finalized = true;
    oStream = null;
  }
}
 

int addSomeBlocks(FSDatasetInterface fsdataset, int startingBlockId) throws IOException {
  int bytesAdded = 0;
  for (int i = startingBlockId; i < startingBlockId+NUMBLOCKS; ++i) {
    Block b = new Block(i, 0, 0); // we pass expected len as zero, - fsdataset should use the sizeof actual data written
    OutputStream dataOut  = fsdataset.writeToBlock(0, b, false, false).dataOut;
    assertEquals(0, fsdataset.getFinalizedBlockLength(0,b));
    for (int j=1; j <= blockIdToLen(i); ++j) {
      dataOut.write(j);
      assertEquals(j, fsdataset.getFinalizedBlockLength(0,b)); // correct length even as we write
      bytesAdded++;
    }
    dataOut.close();
    b.setNumBytes(blockIdToLen(i));
    fsdataset.finalizeBlock(0,b);
    assertEquals(blockIdToLen(i), fsdataset.getFinalizedBlockLength(0,b));
  }
  return bytesAdded;  
}
 

/** {@inheritDoc} */
public Block getStoredBlock(int namespaceId, long blkid,
    boolean useOnDiskLength) throws IOException {
  lock.readLock().lock();
  try {
    File blockfile = findBlockFile(namespaceId, blkid);
    if (blockfile == null) {
      return null;
    }
    File metafile = findMetaFile(blockfile, true);
    if (metafile == null) {
      return null;
    }
    Block block = new Block(blkid);
    if (useOnDiskLength) {
      block.setNumBytes(getOnDiskLength(namespaceId, block));
    } else {
      block.setNumBytes(getVisibleLength(namespaceId, block));
    }
    block.setGenerationStamp(parseGenerationStamp(blockfile, metafile));
    return block;
  } finally {
    lock.readLock().unlock();
  }
}
 

BInfo(Block b, boolean forWriting) throws IOException {
  theBlock = new Block(b);
  if (theBlock.getNumBytes() < 0) {
    theBlock.setNumBytes(0);
  }
  if (!storage.alloc(theBlock.getNumBytes())) { // expected length - actual length may
                                      // be more - we find out at finalize
    DataNode.LOG.warn("Lack of free storage on a block alloc");
    throw new IOException("Creating block, no free space available");
  }

  if (forWriting) {
    finalized = false;
    oStream = new SimulatedOutputStream();
  } else {
    finalized = true;
    oStream = null;
  }
}
 

int addSomeBlocks(FSDatasetInterface fsdataset, int startingBlockId) throws IOException {
  int bytesAdded = 0;
  for (int i = startingBlockId; i < startingBlockId+NUMBLOCKS; ++i) {
    Block b = new Block(i, 0, 0); // we pass expected len as zero, - fsdataset should use the sizeof actual data written
    OutputStream dataOut  = fsdataset.writeToBlock(b, false).dataOut;
    assertEquals(0, fsdataset.getLength(b));
    for (int j=1; j <= blockIdToLen(i); ++j) {
      dataOut.write(j);
      assertEquals(j, fsdataset.getLength(b)); // correct length even as we write
      bytesAdded++;
    }
    dataOut.close();
    b.setNumBytes(blockIdToLen(i));
    fsdataset.finalizeBlock(b);
    assertEquals(blockIdToLen(i), fsdataset.getLength(b));
  }
  return bytesAdded;  
}
 

/**
 * Add a new block into the given INodeFile
 */
private void addNewBlock(FSDirectory fsDir, AddBlockOp op, INodeFile file)
    throws IOException {
  BlockInfoContiguous[] oldBlocks = file.getBlocks();
  Block pBlock = op.getPenultimateBlock();
  Block newBlock= op.getLastBlock();
  
  if (pBlock != null) { // the penultimate block is not null
    Preconditions.checkState(oldBlocks != null && oldBlocks.length > 0);
    // compare pBlock with the last block of oldBlocks
    Block oldLastBlock = oldBlocks[oldBlocks.length - 1];
    if (oldLastBlock.getBlockId() != pBlock.getBlockId()
        || oldLastBlock.getGenerationStamp() != pBlock.getGenerationStamp()) {
      throw new IOException(
          "Mismatched block IDs or generation stamps for the old last block of file "
              + op.getPath() + ", the old last block is " + oldLastBlock
              + ", and the block read from editlog is " + pBlock);
    }
    
    oldLastBlock.setNumBytes(pBlock.getNumBytes());
    if (oldLastBlock instanceof BlockInfoContiguousUnderConstruction) {
      fsNamesys.getBlockManager().forceCompleteBlock(file,
          (BlockInfoContiguousUnderConstruction) oldLastBlock);
      fsNamesys.getBlockManager().processQueuedMessagesForBlock(pBlock);
    }
  } else { // the penultimate block is null
    Preconditions.checkState(oldBlocks == null || oldBlocks.length == 0);
  }
  // add the new block
  BlockInfoContiguous newBI = new BlockInfoContiguousUnderConstruction(
        newBlock, file.getBlockReplication());
  fsNamesys.getBlockManager().addBlockCollection(newBI, file);
  file.addBlock(newBI);
  fsNamesys.getBlockManager().processQueuedMessagesForBlock(newBlock);
}
 

/**
 * Add a new block into the given INodeFile
 */
private void addNewBlock(FSDirectory fsDir, AddBlockOp op, INodeFile file)
    throws IOException {
  BlockInfoContiguous[] oldBlocks = file.getBlocks();
  Block pBlock = op.getPenultimateBlock();
  Block newBlock= op.getLastBlock();
  
  if (pBlock != null) { // the penultimate block is not null
    Preconditions.checkState(oldBlocks != null && oldBlocks.length > 0);
    // compare pBlock with the last block of oldBlocks
    Block oldLastBlock = oldBlocks[oldBlocks.length - 1];
    if (oldLastBlock.getBlockId() != pBlock.getBlockId()
        || oldLastBlock.getGenerationStamp() != pBlock.getGenerationStamp()) {
      throw new IOException(
          "Mismatched block IDs or generation stamps for the old last block of file "
              + op.getPath() + ", the old last block is " + oldLastBlock
              + ", and the block read from editlog is " + pBlock);
    }
    
    oldLastBlock.setNumBytes(pBlock.getNumBytes());
    if (oldLastBlock instanceof BlockInfoContiguousUnderConstruction) {
      fsNamesys.getBlockManager().forceCompleteBlock(file,
          (BlockInfoContiguousUnderConstruction) oldLastBlock);
      fsNamesys.getBlockManager().processQueuedMessagesForBlock(pBlock);
    }
  } else { // the penultimate block is null
    Preconditions.checkState(oldBlocks == null || oldBlocks.length == 0);
  }
  // add the new block
  BlockInfoContiguous newBI = new BlockInfoContiguousUnderConstruction(
        newBlock, file.getBlockReplication());
  fsNamesys.getBlockManager().addBlockCollection(newBI, file);
  file.addBlock(newBI);
  fsNamesys.getBlockManager().processQueuedMessagesForBlock(newBlock);
}
 

/** {@inheritDoc} */
public Block getStoredBlock(int namespaceId, long blkid) throws IOException {
  Block b = new Block(blkid);
  BInfo binfo = getBlockMap(namespaceId).get(b);
  if (binfo == null) {
    return null;
  }
  b.setGenerationStamp(binfo.getGenerationStamp());
  b.setNumBytes(binfo.getlength());
  return b;
}
 

/** {@inheritDoc} */
public Block getStoredBlock(long blkid) throws IOException {
  Block b = new Block(blkid);
  BInfo binfo = blockMap.get(b);
  if (binfo == null) {
    return null;
  }
  b.setGenerationStamp(binfo.getGenerationStamp());
  b.setNumBytes(binfo.getlength());
  return b;
}
 

/**
 * @inheritDoc
 */
public long[] blockReceivedAndDeletedNew(DatanodeRegistration nodeReg,
      IncrementalBlockReport receivedAndDeletedBlocks) throws IOException {
  long[] failedMap = null;
  if (runInfo.shutdown || !runInfo.isRunning) {
    // Do not attempt to process blocks when
    // the namenode is not running
    if (currentAvatar == Avatar.STANDBY) {
      return new long[0];
    } else {
      return null;
    }
  }
  HashSet<Long> failedIds;
  if (currentAvatar == Avatar.STANDBY) {
    int noAck = receivedAndDeletedBlocks.getLength();
    
    // retry all block if the standby is behind consuming edits
    if (ignoreDatanodes()) {
      LOG.info("Standby fell behind. Telling " + nodeReg.toString() +
      " to retry incremental block report of " + noAck
      + " blocks later.");
      failedMap = LightWeightBitSet.getBitSet(noAck);
      for (int i = 0; i < noAck; i++)
        LightWeightBitSet.set(failedMap, i);
      return failedMap;
    }
    
    Block blockRD = new Block();
    failedIds = new HashSet<Long>();
    failedMap = LightWeightBitSet.getBitSet(noAck);
    namesystem.writeLock();
    try {
      receivedAndDeletedBlocks.resetIterator();
      for (int currentBlock = 0; currentBlock < noAck; currentBlock++) {
        receivedAndDeletedBlocks.getNext(blockRD);
        if(failedIds.contains(blockRD.getBlockId())){
          // check if there was no other blocking failed request
          blockRD.setNumBytes(BlockFlags.IGNORE);
          receivedAndDeletedBlocks.setBlock(blockRD, currentBlock);
          LightWeightBitSet.set(failedMap, currentBlock);
          continue;
        }
        BlockInfo storedBlock = namesystem.blocksMap.getStoredBlock(blockRD);
        if (!DFSUtil.isDeleted(blockRD) && (storedBlock == null) &&
            (!namesystem.getPersistBlocks() ||
            blockRD.getGenerationStamp() >= namesystem.getGenerationStamp())) {
          // If this block does not belong to anyfile and its GS
          // is no less than the avatar node's GS,
          // AvatarNode may not consume the file/block creation edit log yet,
          // so adding it to the failed list.
          // - do not process any requestes for blocks with the same block id
          // (also add them to the failed list.
          // - do not block other requests
          blockRD.setNumBytes(BlockFlags.IGNORE);
          receivedAndDeletedBlocks.setBlock(blockRD, currentBlock);
          LightWeightBitSet.set(failedMap, currentBlock);
          failedIds.add(blockRD.getBlockId());
        }
      }
    } finally {
      namesystem.writeUnlock();
      if (failedMap != null && LightWeightBitSet.cardinality(failedMap) != 0) {
        LOG.info("*BLOCK* NameNode.blockReceivedAndDeleted: "
          + "from " + nodeReg.getName() + " has to retry "
          + LightWeightBitSet.cardinality(failedMap) + " blocks.");
      }
      receivedAndDeletedBlocks.resetIterator();
      for (int currentBlock = 0; currentBlock < noAck; currentBlock++) {
        receivedAndDeletedBlocks.getNext(blockRD);
        if (!LightWeightBitSet.get(failedMap, currentBlock))
          continue;
        LOG.info("blockReceivedDeleted " + (DFSUtil.isDeleted(blockRD) ? "DELETED" : "RECEIVED")
            + " request received for "
            + blockRD + " on " + nodeReg.getName() + " size "
            + blockRD.getNumBytes()
            + " But it does not belong to any file." + " Retry later.");
      }
    }
  }
  super.blockReceivedAndDeleted(nodeReg, receivedAndDeletedBlocks);
  return failedMap;
}
 

public static void markAsDeleted(Block block) {
  block.setNumBytes(BlockFlags.DELETED);
}