Java Code Examples for org.apache.hadoop.io.DataInputBuffer#reset()

/**
 * A utility that tests serialization/deserialization. 
 * @param conf configuration to use, "io.serializations" is read to 
 * determine the serialization
 * @param <K> the class of the item
 * @param before item to (de)serialize
 * @return deserialized item
 */
public static <K> K testSerialization(Configuration conf, K before)
	throws Exception {

  SerializationFactory factory = new SerializationFactory(conf);
  Serializer<K> serializer 
    = factory.getSerializer(GenericsUtil.getClass(before));
  Deserializer<K> deserializer 
    = factory.getDeserializer(GenericsUtil.getClass(before));

  DataOutputBuffer out = new DataOutputBuffer();
  serializer.open(out);
  serializer.serialize(before);
  serializer.close();

  DataInputBuffer in = new DataInputBuffer();
  in.reset(out.getData(), out.getLength());
  deserializer.open(in);
  K after = deserializer.deserialize(null);
  deserializer.close();
  return after;
}
 

@Test
public void testWriteRead() {
  mfDatasetInputSplit.addDataset("dataSet1");
  mfDatasetInputSplit.addDataset("dataSet2");
  DataOutputBuffer dob = new DataOutputBuffer();
  DataInputBuffer dib = new DataInputBuffer();
  MainframeDatasetInputSplit mfReader = new MainframeDatasetInputSplit();
  try {
    mfDatasetInputSplit.write(dob);
    dib.reset(dob.getData(), dob.getLength());
    mfReader.readFields(dib);
    Assert.assertNotNull("MFReader get data from tester", mfReader);
    Assert.assertEquals(2, mfReader.getLength());
    Assert.assertEquals("dataSet1", mfReader.getNextDataset());
    Assert.assertEquals("dataSet2", mfReader.getNextDataset());
  } catch (IOException ioe) {
    Assert.fail("No IOException should be thrown!");
  } catch (InterruptedException ie) {
    Assert.fail("No InterruptedException should be thrown!");
  }
}
 

@Override
public void writeUncompressedBytes(DataOutputStream outStream)
  throws IOException {
  if (decompressedStream == null) {
    rawData = new DataInputBuffer();
    decompressedStream = codec.createInputStream(rawData);
  } else {
    decompressedStream.resetState();
  }
  rawData.reset(data, 0, dataSize);

  byte[] buffer = new byte[8192];
  int bytesRead = 0;
  while ((bytesRead = decompressedStream.read(buffer, 0, 8192)) != -1) {
    outStream.write(buffer, 0, bytesRead);
  }
}
 

@Test
public void testAMRMTokenIdentifier() throws IOException {
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(1, 1), 1);
  int masterKeyId = 1;

  AMRMTokenIdentifier token = new AMRMTokenIdentifier(appAttemptId, masterKeyId);
  
  AMRMTokenIdentifier anotherToken = new AMRMTokenIdentifier();
  byte[] tokenContent = token.getBytes();
  DataInputBuffer dib = new DataInputBuffer();
  dib.reset(tokenContent, tokenContent.length);
  anotherToken.readFields(dib);
      
  // verify the whole record equals with original record
  Assert.assertEquals("Token is not the same after serialization " +
      "and deserialization.", token, anotherToken);
      
  Assert.assertEquals("ApplicationAttemptId from proto is not the same with original token",
      anotherToken.getApplicationAttemptId(), appAttemptId);
  
  Assert.assertEquals("masterKeyId from proto is not the same with original token",
      anotherToken.getKeyId(), masterKeyId);
}
 

@Override
public DataInputBuffer getKey() throws IOException {
  ByteArrayOutputStream dt = new ByteArrayOutputStream();
  GridmixKey key = new GridmixKey(GridmixKey.REDUCE_SPEC, 10 * counter, 1L);
  Spec spec = new Spec();
  spec.rec_in = counter;
  spec.rec_out = counter;
  spec.bytes_out = counter * 100;

  key.setSpec(spec);
  key.write(new DataOutputStream(dt));
  DataInputBuffer result = new DataInputBuffer();
  byte[] b = dt.toByteArray();
  result.reset(b, 0, b.length);
  return result;
}
 

public void nextRawValue(DataInputBuffer value) throws IOException {
  try {
    int pos = memDataIn.getPosition();
    byte[] data = memDataIn.getData();
    value.reset(data, pos, currentValueLength);

    // Position for the next record
    long skipped = memDataIn.skip(currentValueLength);
    if (skipped != currentValueLength) {
      throw new IOException("Rec# " + recNo + 
          ": Failed to skip past value of length: " + 
          currentValueLength);
    }
    // Record the byte
    bytesRead += currentValueLength;

    ++recNo;
  } catch (IOException ioe) {
    dumpOnError();
    throw ioe;
  }
}
 

public void testIFileStream() throws Exception {
  final int DLEN = 100;
  DataOutputBuffer dob = new DataOutputBuffer(DLEN + 4);
  IFileOutputStream ifos = new IFileOutputStream(dob);
  for (int i = 0; i < DLEN; ++i) {
    ifos.write(i);
  }
  ifos.close();
  DataInputBuffer dib = new DataInputBuffer();
  dib.reset(dob.getData(), DLEN + 4);
  IFileInputStream ifis = new IFileInputStream(dib, 104, new Configuration());
  for (int i = 0; i < DLEN; ++i) {
    assertEquals(i, ifis.read());
  }
  ifis.close();
}
 

private Writer writeTestFileUsingDataBuffer(Writer writer, boolean repeatKeys,
    List<KVPair> data) throws IOException {
  DataInputBuffer previousKey = new DataInputBuffer();
  DataInputBuffer key = new DataInputBuffer();
  DataInputBuffer value = new DataInputBuffer();
  for (KVPair kvp : data) {
    populateData(kvp, key, value);

    if (repeatKeys && (previousKey != null && BufferUtils.compare(key, previousKey) == 0)) {
      writer.append(IFile.REPEAT_KEY, value);
    } else {
      writer.append(key, value);
    }
    previousKey.reset(key.getData(), 0, key.getLength());
  }

  writer.close();

  LOG.info("Uncompressed: " + writer.getRawLength());
  LOG.info("CompressedSize: " + writer.getCompressedLength());

  return writer;
}
 

public void testIFileStream() throws Exception {
  final int DLEN = 100;
  DataOutputBuffer dob = new DataOutputBuffer(DLEN + 4);
  IFileOutputStream ifos = new IFileOutputStream(dob);
  for (int i = 0; i < DLEN; ++i) {
    ifos.write(i);
  }
  ifos.close();
  DataInputBuffer dib = new DataInputBuffer();
  dib.reset(dob.getData(), DLEN + 4);
  IFileInputStream ifis = new IFileInputStream(dib, 104);
  for (int i = 0; i < DLEN; ++i) {
    assertEquals(i, ifis.read());
  }
  ifis.close();
}
 

public boolean next(DataInputBuffer key, DataInputBuffer value)
    throws IOException {
  MemoryBlockIndex memBlkIdx = keyValueIterator.next();
  if (memBlkIdx != null) {
    int pos = memBlkIdx.getIndex();
    MemoryBlock memBlk = memBlkIdx.getMemoryBlock();
    int offset = memBlk.offsets[pos];
    int keyLen = memBlk.keyLenArray[pos];
    int valLen = memBlk.valueLenArray[pos];
    dataOutputBuffer.reset();
    dataOutputBuffer.writeInt(keyLen);
    dataOutputBuffer.write(kvbuffer, offset, keyLen);
    dataOutputBuffer.writeInt(valLen);
    dataOutputBuffer.write(kvbuffer, offset + keyLen, valLen);
    key.reset(dataOutputBuffer.getData(), 0, keyLen
        + WritableUtils.INT_LENGTH_BYTES);
    value.reset(dataOutputBuffer.getData(), keyLen
        + WritableUtils.INT_LENGTH_BYTES, valLen
        + WritableUtils.INT_LENGTH_BYTES);
    return true;
  }
  return false;
}
 

public String[] getSorted() throws IOException {
  String[] ret = new String[indices.length];
  Text t = new Text();
  DataInputBuffer dib = new DataInputBuffer();
  for (int i = 0; i < ret.length; ++i) {
    int ii = indices[i];
    dib.reset(bytes, offsets[ii],
    ((ii + 1 == indices.length) ? eob : offsets[ii + 1]) - offsets[ii]);
    t.readFields(dib);
    ret[i] = t.toString();
  }
  return ret;
}
 

@Override
public DataInputBuffer getKey() throws IOException {
  DataInputBuffer key = new DataInputBuffer();
  Text text = new Text(keys[i]);
  DataOutputBuffer out = new DataOutputBuffer();
  text.write(out);
  key.reset(out.getData(), out.getLength());
  return key;
}
 

public KeyState readRawKey(DataInputBuffer key) throws IOException {
  try {
    if (!positionToNextRecord(memDataIn)) {
      return KeyState.NO_KEY;
    }
    // Setup the key
    int pos = memDataIn.getPosition();
    byte[] data = memDataIn.getData();
    if (currentKeyLength == IFile.RLE_MARKER) {
      // get key length from original key
      key.reset(data, originalKeyPos, originalKeyLength);
      return KeyState.SAME_KEY;
    }
    key.reset(data, pos, currentKeyLength);
    // Position for the next value
    long skipped = memDataIn.skip(currentKeyLength);
    if (skipped != currentKeyLength) {
      throw new IOException("Rec# " + recNo +
          ": Failed to skip past key of length: " +
          currentKeyLength);
    }
    bytesRead += currentKeyLength;
    return KeyState.NEW_KEY;
  } catch (IOException ioe) {
    dumpOnError();
    throw ioe;
  }
}
 

private void writePartition(int pos, WrappedBuffer wrappedBuffer, Writer writer,
    DataInputBuffer keyBuffer, DataInputBuffer valBuffer) throws IOException {
  while (pos != WrappedBuffer.PARTITION_ABSENT_POSITION) {
    int metaIndex = pos / INT_SIZE;
    int keyLength = wrappedBuffer.metaBuffer.get(metaIndex + INDEX_KEYLEN);
    int valLength = wrappedBuffer.metaBuffer.get(metaIndex + INDEX_VALLEN);
    keyBuffer.reset(wrappedBuffer.buffer, pos + META_SIZE, keyLength);
    valBuffer.reset(wrappedBuffer.buffer, pos + META_SIZE + keyLength, valLength);

    writer.append(keyBuffer, valBuffer);
    pos = wrappedBuffer.metaBuffer.get(metaIndex + INDEX_NEXT);
  }
}
 

static void lengthTest(GridmixRecord x, GridmixRecord y, int min,
    int max) throws Exception {
  final Random r = new Random();
  final long seed = r.nextLong();
  r.setSeed(seed);
  LOG.info("length: " + seed);
  final DataInputBuffer in = new DataInputBuffer();
  final DataOutputBuffer out1 = new DataOutputBuffer();
  final DataOutputBuffer out2 = new DataOutputBuffer();
  for (int i = min; i < max; ++i) {
    setSerialize(x, r.nextLong(), i, out1);
    // check write
    assertEquals(i, out1.getLength());
    // write to stream
    x.write(out2);
    // check read
    in.reset(out1.getData(), 0, out1.getLength());
    y.readFields(in);
    assertEquals(i, x.getSize());
    assertEquals(i, y.getSize());
  }
  // check stream read
  in.reset(out2.getData(), 0, out2.getLength());
  for (int i = min; i < max; ++i) {
    y.readFields(in);
    assertEquals(i, y.getSize());
  }
}
 

private void populateData(KVPair kvp, DataInputBuffer key, DataInputBuffer value)
    throws  IOException {
  DataOutputBuffer k = new DataOutputBuffer();
  DataOutputBuffer v = new DataOutputBuffer();
  kvp.getKey().write(k);
  kvp.getvalue().write(v);
  key.reset(k.getData(), 0, k.getLength());
  value.reset(v.getData(), 0, v.getLength());
}
 

public boolean nextRawKey(DataInputBuffer key) throws IOException {
  if (kvIter.next()) {
    final DataInputBuffer kb = kvIter.getKey();
    final int kp = kb.getPosition();
    final int klen = kb.getLength() - kp;
    key.reset(kb.getData(), kp, klen);
    bytesRead += klen;
    return true;
  }
  return false;
}
 

@Test
public void testRMDelegationTokenIdentifier() throws IOException {
  
  Text owner = new Text("user1");
  Text renewer = new Text("user2");
  Text realUser = new Text("user3");
  long issueDate = 1;
  long maxDate = 2;
  int sequenceNumber = 3;
  int masterKeyId = 4;
  
  RMDelegationTokenIdentifier token = 
      new RMDelegationTokenIdentifier(owner, renewer, realUser);
  token.setIssueDate(issueDate);
  token.setMaxDate(maxDate);
  token.setSequenceNumber(sequenceNumber);
  token.setMasterKeyId(masterKeyId);
  
  RMDelegationTokenIdentifier anotherToken = new RMDelegationTokenIdentifier();
  
  byte[] tokenContent = token.getBytes();
  DataInputBuffer dib = new DataInputBuffer();
  dib.reset(tokenContent, tokenContent.length);
  anotherToken.readFields(dib);
  dib.close();
  // verify the whole record equals with original record
  Assert.assertEquals("Token is not the same after serialization " +
      "and deserialization.", token, anotherToken);
  
  Assert.assertEquals("owner from proto is not the same with original token",
      anotherToken.getOwner(), owner);
  
  Assert.assertEquals("renewer from proto is not the same with original token",
      anotherToken.getRenewer(), renewer);
  
  Assert.assertEquals("realUser from proto is not the same with original token",
      anotherToken.getRealUser(), realUser);
  
  Assert.assertEquals("issueDate from proto is not the same with original token",
      anotherToken.getIssueDate(), issueDate);
  
  Assert.assertEquals("maxDate from proto is not the same with original token",
      anotherToken.getMaxDate(), maxDate);
  
  Assert.assertEquals("sequenceNumber from proto is not the same with original token",
      anotherToken.getSequenceNumber(), sequenceNumber);
  
  Assert.assertEquals("masterKeyId from proto is not the same with original token",
      anotherToken.getMasterKeyId(), masterKeyId);
  
  // Test getProto    
  RMDelegationTokenIdentifier token1 = 
      new RMDelegationTokenIdentifier(owner, renewer, realUser);
  token1.setIssueDate(issueDate);
  token1.setMaxDate(maxDate);
  token1.setSequenceNumber(sequenceNumber);
  token1.setMasterKeyId(masterKeyId);
  YARNDelegationTokenIdentifierProto tokenProto = token1.getProto();
  // Write token proto to stream
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  DataOutputStream out = new DataOutputStream(baos);
  tokenProto.writeTo(out);

  // Read token
  byte[] tokenData = baos.toByteArray();
  RMDelegationTokenIdentifier readToken = new RMDelegationTokenIdentifier();
  DataInputBuffer db = new DataInputBuffer();
  db.reset(tokenData, tokenData.length);
  readToken.readFields(db);

  // Verify if read token equals with original token
  Assert.assertEquals("Token from getProto is not the same after " +
      "serialization and deserialization.", token1, readToken);
  db.close();
  out.close();
}
 

private static void codecTest(Configuration conf, int seed, int count, 
                              String codecClass) 
  throws IOException {
  
  // Create the codec
  CompressionCodec codec = null;
  try {
    codec = (CompressionCodec)
      ReflectionUtils.newInstance(conf.getClassByName(codecClass), conf);
  } catch (ClassNotFoundException cnfe) {
    throw new IOException("Illegal codec!");
  }
  LOG.info("Created a Codec object of type: " + codecClass);

  // Generate data
  DataOutputBuffer data = new DataOutputBuffer();
  RandomDatum.Generator generator = new RandomDatum.Generator(seed);
  for(int i=0; i < count; ++i) {
    generator.next();
    RandomDatum key = generator.getKey();
    RandomDatum value = generator.getValue();
    
    key.write(data);
    value.write(data);
  }
  DataInputBuffer originalData = new DataInputBuffer();
  DataInputStream originalIn = new DataInputStream(new BufferedInputStream(originalData));
  originalData.reset(data.getData(), 0, data.getLength());
  
  LOG.info("Generated " + count + " records");
  
  // Compress data
  DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
  CompressionOutputStream deflateFilter = 
    codec.createOutputStream(compressedDataBuffer);
  DataOutputStream deflateOut = 
    new DataOutputStream(new BufferedOutputStream(deflateFilter));
  deflateOut.write(data.getData(), 0, data.getLength());
  deflateOut.flush();
  deflateFilter.finish();
  LOG.info("Finished compressing data");
  
  // De-compress data
  DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
  deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0, 
                               compressedDataBuffer.getLength());
  CompressionInputStream inflateFilter = 
    codec.createInputStream(deCompressedDataBuffer);
  DataInputStream inflateIn = 
    new DataInputStream(new BufferedInputStream(inflateFilter));

  // Check
  for(int i=0; i < count; ++i) {
    RandomDatum k1 = new RandomDatum();
    RandomDatum v1 = new RandomDatum();
    k1.readFields(originalIn);
    v1.readFields(originalIn);
    
    RandomDatum k2 = new RandomDatum();
    RandomDatum v2 = new RandomDatum();
    k2.readFields(inflateIn);
    v2.readFields(inflateIn);
  }
  LOG.info("SUCCESS! Completed checking " + count + " records");
}
 

@Test
public void testTimelineDelegationTokenIdentifier() throws IOException {
  
  Text owner = new Text("user1");
  Text renewer = new Text("user2");
  Text realUser = new Text("user3");
  long issueDate = 1;
  long maxDate = 2;
  int sequenceNumber = 3;
  int masterKeyId = 4;
  
  TimelineDelegationTokenIdentifier token = 
      new TimelineDelegationTokenIdentifier(owner, renewer, realUser);
  token.setIssueDate(issueDate);
  token.setMaxDate(maxDate);
  token.setSequenceNumber(sequenceNumber);
  token.setMasterKeyId(masterKeyId);
  
  TimelineDelegationTokenIdentifier anotherToken = 
      new TimelineDelegationTokenIdentifier();
  
  byte[] tokenContent = token.getBytes();
  DataInputBuffer dib = new DataInputBuffer();
  dib.reset(tokenContent, tokenContent.length);
  anotherToken.readFields(dib);
      
  // verify the whole record equals with original record
  Assert.assertEquals("Token is not the same after serialization " +
      "and deserialization.", token, anotherToken);
  
  Assert.assertEquals("owner from proto is not the same with original token",
      anotherToken.getOwner(), owner);
  
  Assert.assertEquals("renewer from proto is not the same with original token",
      anotherToken.getRenewer(), renewer);
  
  Assert.assertEquals("realUser from proto is not the same with original token",
      anotherToken.getRealUser(), realUser);
  
  Assert.assertEquals("issueDate from proto is not the same with original token",
      anotherToken.getIssueDate(), issueDate);
  
  Assert.assertEquals("maxDate from proto is not the same with original token",
      anotherToken.getMaxDate(), maxDate);
  
  Assert.assertEquals("sequenceNumber from proto is not the same with original token",
      anotherToken.getSequenceNumber(), sequenceNumber);
  
  Assert.assertEquals("masterKeyId from proto is not the same with original token",
      anotherToken.getMasterKeyId(), masterKeyId);
}