org.apache.hadoop.fs.FSInputChecker Java Examples

The following examples show how to use org.apache.hadoop.fs.FSInputChecker. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BlockXCodingMerger.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
/**
 * reads in the partial crc chunk and computes checksum of pre-existing data
 * in partial chunk.
 */
private void computePartialChunkCrc(long blkoff, long ckoff,
		int bytesPerChecksum) throws IOException {

	// find offset of the beginning of partial chunk.
	//
	int sizePartialChunk = (int) (blkoff % bytesPerChecksum);
	int checksumSize = checksum.getChecksumSize();
	blkoff = blkoff - sizePartialChunk;

	// create an input stream from the block file
	// and read in partial crc chunk into temporary buffer
	byte[] buf = new byte[sizePartialChunk];
	byte[] crcbuf = new byte[checksumSize];
	FSDataset.BlockInputStreams instr = null;
	try {
		instr = datanode.data.getTmpInputStreams(namespaceId, block,
				blkoff, ckoff);
		IOUtils.readFully(instr.dataIn, buf, 0, sizePartialChunk);

		// open meta file and read in crc value computer earlier
		IOUtils.readFully(instr.checksumIn, crcbuf, 0, crcbuf.length);
	} finally {
		IOUtils.closeStream(instr);
	}

	// compute crc of partial chunk from data read in the block file.
	partialCrc = new CRC32();
	partialCrc.update(buf, 0, sizePartialChunk);

	// paranoia! verify that the pre-computed crc matches what we
	// recalculated just now
	if (partialCrc.getValue() != FSInputChecker.checksum2long(crcbuf)) {
		String msg = "Partial CRC " + partialCrc.getValue()
				+ " does not match value computed the "
				+ " last time file was closed "
				+ FSInputChecker.checksum2long(crcbuf);
		throw new IOException(msg);
	}
}
Example #2
Source File: BlockReceiver.java    From RDFS with Apache License 2.0 4 votes vote down vote up 
/**
 * reads in the partial crc chunk and computes checksum
 * of pre-existing data in partial chunk.
 */
private void computePartialChunkCrc(long blkoff, long ckoff, 
                                    int bytesPerChecksum) throws IOException {

  // find offset of the beginning of partial chunk.
  //
  int sizePartialChunk = (int) (blkoff % bytesPerChecksum);
  int checksumSize = checksum.getChecksumSize();
  blkoff = blkoff - sizePartialChunk;
  LOG.info("computePartialChunkCrc sizePartialChunk " + 
            sizePartialChunk +
            " block " + block +
            " offset in block " + blkoff +
            " offset in metafile " + ckoff);

  // create an input stream from the block file
  // and read in partial crc chunk into temporary buffer
  //
  byte[] buf = new byte[sizePartialChunk];
  byte[] crcbuf = new byte[checksumSize];
  FSDataset.BlockInputStreams instr = null;
  try { 
    instr = datanode.data.getTmpInputStreams(namespaceId, block, blkoff, ckoff);
    IOUtils.readFully(instr.dataIn, buf, 0, sizePartialChunk);

    // open meta file and read in crc value computer earlier
    IOUtils.readFully(instr.checksumIn, crcbuf, 0, crcbuf.length);
  } finally {
    IOUtils.closeStream(instr);
  }

  // compute crc of partial chunk from data read in the block file.
  partialCrc = new CRC32();
  partialCrc.update(buf, 0, sizePartialChunk);
  LOG.info("Read in partial CRC chunk from disk for block " + block);

  // paranoia! verify that the pre-computed crc matches what we
  // recalculated just now
  if (partialCrc.getValue() != FSInputChecker.checksum2long(crcbuf)) {
    String msg = "Partial CRC " + partialCrc.getValue() +
                 " does not match value computed the " +
                 " last time file was closed " +
                 FSInputChecker.checksum2long(crcbuf);
    throw new IOException(msg);
  }
  //LOG.debug("Partial CRC matches 0x" + 
  //            Long.toHexString(partialCrc.getValue()));
}
Example #3
Source File: BlockReceiver.java    From hadoop-gpu with Apache License 2.0 4 votes vote down vote up 
/**
 * reads in the partial crc chunk and computes checksum
 * of pre-existing data in partial chunk.
 */
private void computePartialChunkCrc(long blkoff, long ckoff, 
                                    int bytesPerChecksum) throws IOException {

  // find offset of the beginning of partial chunk.
  //
  int sizePartialChunk = (int) (blkoff % bytesPerChecksum);
  int checksumSize = checksum.getChecksumSize();
  blkoff = blkoff - sizePartialChunk;
  LOG.info("computePartialChunkCrc sizePartialChunk " + 
            sizePartialChunk +
            " block " + block +
            " offset in block " + blkoff +
            " offset in metafile " + ckoff);

  // create an input stream from the block file
  // and read in partial crc chunk into temporary buffer
  //
  byte[] buf = new byte[sizePartialChunk];
  byte[] crcbuf = new byte[checksumSize];
  FSDataset.BlockInputStreams instr = null;
  try { 
    instr = datanode.data.getTmpInputStreams(block, blkoff, ckoff);
    IOUtils.readFully(instr.dataIn, buf, 0, sizePartialChunk);

    // open meta file and read in crc value computer earlier
    IOUtils.readFully(instr.checksumIn, crcbuf, 0, crcbuf.length);
  } finally {
    IOUtils.closeStream(instr);
  }

  // compute crc of partial chunk from data read in the block file.
  partialCrc = new CRC32();
  partialCrc.update(buf, 0, sizePartialChunk);
  LOG.info("Read in partial CRC chunk from disk for block " + block);

  // paranoia! verify that the pre-computed crc matches what we
  // recalculated just now
  if (partialCrc.getValue() != FSInputChecker.checksum2long(crcbuf)) {
    String msg = "Partial CRC " + partialCrc.getValue() +
                 " does not match value computed the " +
                 " last time file was closed " +
                 FSInputChecker.checksum2long(crcbuf);
    throw new IOException(msg);
  }
  //LOG.debug("Partial CRC matches 0x" + 
  //            Long.toHexString(partialCrc.getValue()));
}