org.apache.hadoop.util.PureJavaCrc32 Java Examples

private static void createIndexFile(File indexFile, Configuration conf)
    throws IOException {
  if (indexFile.exists()) {
    System.out.println("Deleting existing file");
    indexFile.delete();
  }
  indexFile.createNewFile();
  FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append(
      new Path(indexFile.getAbsolutePath()));
  Checksum crc = new PureJavaCrc32();
  crc.reset();
  CheckedOutputStream chk = new CheckedOutputStream(output, crc);
  String msg = "Writing new index file. This file will be used only " +
      "for the testing.";
  chk.write(Arrays.copyOf(msg.getBytes(),
      MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
  output.writeLong(chk.getChecksum().getValue());
  output.close();
}
 

public BlockXCodingMerger(Block block, int namespaceId,
		DataInputStream[] childInputStreams, long offsetInBlock,
		long length, String[] childAddrs, String myAddr,
		DataTransferThrottler throttler,
		int mergerLevel) throws IOException{
	super();
	this.block = block;
	this.namespaceId = namespaceId;
	this.childInputStreams = childInputStreams;
	this.offsetInBlock = offsetInBlock;
	this.length = length;
	this.childAddrs = childAddrs;
	this.myAddr = myAddr;
	this.throttler = throttler;
	this.mergerLevel = mergerLevel;
	Configuration conf = new Configuration();
	this.packetSize = conf.getInt("raid.blockreconstruct.packetsize", 4096);
	this.bytesPerChecksum = conf.getInt("io.bytes.per.checksum", 512);
	this.checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
			bytesPerChecksum, new PureJavaCrc32());
	this.checksumSize = checksum.getChecksumSize();
}
 

private static void createIndexFile(File indexFile, Configuration conf)
    throws IOException {
  if (indexFile.exists()) {
    System.out.println("Deleting existing file");
    indexFile.delete();
  }
  indexFile.createNewFile();
  FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append(
      new Path(indexFile.getAbsolutePath()));
  Checksum crc = new PureJavaCrc32();
  crc.reset();
  CheckedOutputStream chk = new CheckedOutputStream(output, crc);
  String msg = "Writing new index file. This file will be used only " +
      "for the testing.";
  chk.write(Arrays.copyOf(msg.getBytes(),
      MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
  output.writeLong(chk.getChecksum().getValue());
  output.close();
}
 

private static void createIndexFile(File indexFile, Configuration conf)
    throws IOException {
  if (indexFile.exists()) {
    System.out.println("Deleting existing file");
    indexFile.delete();
  }
  Checksum crc = new PureJavaCrc32();
  TezSpillRecord tezSpillRecord = new TezSpillRecord(2);
  tezSpillRecord.putIndex(new TezIndexRecord(0, 10, 10), 0);
  tezSpillRecord.putIndex(new TezIndexRecord(10, 10, 10), 1);
  tezSpillRecord.writeToFile(new Path(indexFile.getAbsolutePath()), conf,
      FileSystem.getLocal(conf).getRaw(), crc);
}
 

public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 

/**
 * object constructor
 */
public BlockReaderAccelerator(
  Configuration conf,
  InetSocketAddress targetAddress,
  DatanodeInfo chosenNode, 
  int dataTransferVersion,
  int namespaceId,
  String clientName,
  Socket sock,
  String hdfsfile, 
  LocatedBlock blk,
  long startOffset, 
  long length,
  boolean verifyChecksum,
  DFSClientMetrics metrics) throws IOException {

  this.conf = conf;
  this.targetAddress = targetAddress;
  this.datanodeInfo = chosenNode;
  this.dataTransferVersion = dataTransferVersion;
  this.namespaceId = namespaceId;
  this.clientName = clientName;
  this.sock = sock;
  this.hdfsfile = hdfsfile;
  this.blk = blk;
  this.startOffset = startOffset;
  this.length = length;
  this.verifyChecksum = verifyChecksum;
  this.metrics = metrics;

   // create a checksum checker
   if (this.verifyChecksum) {
     this.checker = new PureJavaCrc32();
   }
}
 

private DFSOutputStream(DFSClient dfsClient, String src, long blockSize,
    Progressable progress, int bytesPerChecksum, short replication, boolean forceSync,
    boolean doParallelWrites, DatanodeInfo[] favoredNodes)
throws IOException {
  super(new CRC32(), bytesPerChecksum, 4);
  this.dfsClient = dfsClient;
  this.forceSync = forceSync;
  this.doParallelWrites = doParallelWrites;
  this.src = src;
  this.blockSize = blockSize;
  this.blockReplication = replication;
  this.progress = progress;
  
  streamer = new DataStreamer();
  
  packetTimeout =
      dfsClient.conf.getLong("dfs.client.packet.timeout", 15000); // 15 seconds
  // try block recovery 5 times:
  maxRecoveryErrorCount =
      dfsClient.conf.getInt("dfs.client.block.recovery.retries", 5);
  
  if (progress != null) {
    DFSClient.LOG.debug("Set non-null progress callback on DFSOutputStream "+src);
  }

  this.favoredNodes = favoredNodes;

  if ( bytesPerChecksum < 1 || blockSize % bytesPerChecksum != 0) {
    throw new IOException("io.bytes.per.checksum(" + bytesPerChecksum +
                          ") and blockSize(" + blockSize +
                          ") do not match. " + "blockSize should be a " +
                          "multiple of io.bytes.per.checksum");

  }
  checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
                                          bytesPerChecksum,
                                          new PureJavaCrc32());
}
 

/**
 * Create a checksum input stream that reads
 * @param in The input stream to be verified for checksum.
 * @param len The length of the input stream including checksum bytes.
 */
public IFileInputStream(InputStream in, long len) {
  this.in = in;
  sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, 
      Integer.MAX_VALUE, new PureJavaCrc32());
  checksumSize = sum.getChecksumSize();
  length = len;
  dataLength = length - checksumSize;
}
 

/**
 * Create a checksum output stream that writes
 * the bytes to the given stream.
 * @param out
 */
public IFileOutputStream(OutputStream out) {
  super(out);
  sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
      Integer.MAX_VALUE, new PureJavaCrc32());
  barray = new byte[sum.getChecksumSize()];
}
 

public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 

public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 

public static void outputRecords(OutputStream out,
                                 boolean useAscii,
                                 Unsigned16 firstRecordNumber,
                                 Unsigned16 recordsToGenerate,
                                 Unsigned16 checksum
                                 ) throws IOException {
  byte[] row = new byte[100];
  Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
  Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
  Checksum crc = new PureJavaCrc32();
  Unsigned16 tmp = new Unsigned16();
  lastRecordNumber.add(recordsToGenerate);
  Unsigned16 ONE = new Unsigned16(1);
  Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
  while (!recordNumber.equals(lastRecordNumber)) {
    Random16.nextRand(rand);
    if (useAscii) {
      generateAsciiRecord(row, rand, recordNumber);
    } else {
      generateRecord(row, rand, recordNumber);
    }
    if (checksum != null) {
      crc.reset();
      crc.update(row, 0, row.length);
      tmp.set(crc.getValue());
      checksum.add(tmp);
    }
    recordNumber.add(ONE);
    out.write(row);
  }
}
 

protected Checksum initialValue() {
    return new PureJavaCrc32();
}
 

@Test
public void testPureJavaCrc32ByteBuffer() {
  final Checksum expected = new PureJavaCrc32();
  final ChecksumByteBuffer testee = new PureJavaCrc32ByteBuffer();
  new VerifyChecksumByteBuffer(expected, testee).testCorrectness();
}
 

/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, Configuration job, FileSystem fs) throws IOException {
  writeToFile(loc, job, fs, new PureJavaCrc32());
}
 

public TezSpillRecord(Path indexFileName, FileSystem fs, String expectedIndexOwner)
  throws IOException {
  this(indexFileName, fs, new PureJavaCrc32(), expectedIndexOwner);
}
 

public SpillRecord(Path indexFileName, JobConf job, String expectedIndexOwner)
  throws IOException {
  this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}
 

/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, Configuration job)
    throws IOException {
  writeToFile(loc, job, new PureJavaCrc32());
}
 

public TezSpillRecord(Path indexFileName, Configuration job, String expectedIndexOwner)
  throws IOException {
  this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}
 

/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, JobConf job)
    throws IOException {
  writeToFile(loc, job, new PureJavaCrc32());
}
 

protected Checksum initialValue() {
    return new PureJavaCrc32();
}
 

public static BlockReader newBlockReader( int dataTransferVersion,
                                   int namespaceId,
                                   Socket sock, String file,
                                   long blockId,
                                   long genStamp,
                                   long startOffset, long len,
                                   int bufferSize, boolean verifyChecksum,
                                   String clientName, long minSpeedBps)
                                   throws IOException {
  // in and out will be closed when sock is closed (by the caller)
  DataOutputStream out = new DataOutputStream(
    new BufferedOutputStream(NetUtils.getOutputStream(sock,HdfsConstants.WRITE_TIMEOUT)));

  //write the header.
  ReadBlockHeader readBlockHeader = new ReadBlockHeader(
      dataTransferVersion, namespaceId, blockId, genStamp, startOffset, len,
      clientName);
  readBlockHeader.writeVersionAndOpCode(out);
  readBlockHeader.write(out);
  out.flush();

  //
  // Get bytes in block, set streams
  //

  DataInputStream in = new DataInputStream(
      new BufferedInputStream(NetUtils.getInputStream(sock),
                              bufferSize));

  if ( in.readShort() != DataTransferProtocol.OP_STATUS_SUCCESS ) {
    throw new IOException("Got error in response to OP_READ_BLOCK " +
                          "self=" + sock.getLocalSocketAddress() +
                          ", remote=" + sock.getRemoteSocketAddress() +
                          " for file " + file +
                          " for block " + blockId);
  }
  DataChecksum checksum = DataChecksum.newDataChecksum( in , new PureJavaCrc32());
  //Warning when we get CHECKSUM_NULL?

  // Read the first chunk offset.
  long firstChunkOffset = in.readLong();

  if ( firstChunkOffset < 0 || firstChunkOffset > startOffset ||
      firstChunkOffset >= (startOffset + checksum.getBytesPerChecksum())) {
    throw new IOException("BlockReader: error in first chunk offset (" +
                          firstChunkOffset + ") startOffset is " +
                          startOffset + " for file " + file);
  }

  return new BlockReader(file, blockId, in, checksum, verifyChecksum,
      startOffset, firstChunkOffset, sock, minSpeedBps, dataTransferVersion);
}
 

public SpillRecord(Path indexFileName, JobConf job, String expectedIndexOwner)
  throws IOException {
  this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}
 

/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, JobConf job)
    throws IOException {
  writeToFile(loc, job, new PureJavaCrc32());
}
 

/**
 * Write this spill record to the location provided.
 */
public void writeToFile(Path loc, JobConf job)
    throws IOException {
  writeToFile(loc, job, new PureJavaCrc32());
}
 

public SpillRecord(Path indexFileName, JobConf job) throws IOException {
  this(indexFileName, job, new PureJavaCrc32());
}