Java Code Examples for org.apache.hadoop.io.DataInputBuffer#getLength()

The following examples show how to use org.apache.hadoop.io.DataInputBuffer#getLength() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: IFile.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public void append(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  if (keyLength < 0) {
    throw new IOException("Negative key-length not allowed: " + keyLength + 
                          " for " + key);
  }
  
  int valueLength = value.getLength() - value.getPosition();
  if (valueLength < 0) {
    throw new IOException("Negative value-length not allowed: " + 
                          valueLength + " for " + value);
  }

  WritableUtils.writeVInt(out, keyLength);
  WritableUtils.writeVInt(out, valueLength);
  out.write(key.getData(), key.getPosition(), keyLength); 
  out.write(value.getData(), value.getPosition(), valueLength); 

  // Update bytes written
  decompressedBytesWritten += keyLength + valueLength + 
                  WritableUtils.getVIntSize(keyLength) + 
                  WritableUtils.getVIntSize(valueLength);
  ++numRecordsWritten;
}
Example 2
Source File: IFile.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
public void append(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  if (keyLength < 0) {
    throw new IOException("Negative key-length not allowed: " + keyLength + 
                          " for " + key);
  }
  
  int valueLength = value.getLength() - value.getPosition();
  if (valueLength < 0) {
    throw new IOException("Negative value-length not allowed: " + 
                          valueLength + " for " + value);
  }

  WritableUtils.writeVInt(out, keyLength);
  WritableUtils.writeVInt(out, valueLength);
  out.write(key.getData(), key.getPosition(), keyLength); 
  out.write(value.getData(), value.getPosition(), valueLength); 

  // Update bytes written
  decompressedBytesWritten += keyLength + valueLength + 
                  WritableUtils.getVIntSize(keyLength) + 
                  WritableUtils.getVIntSize(valueLength);
  ++numRecordsWritten;
}
Example 3
Source File: ReduceTask.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
public boolean next(DataInputBuffer key, DataInputBuffer value)
    throws IOException {
  if (kvIter.next()) {
    final DataInputBuffer kb = kvIter.getKey();
    final DataInputBuffer vb = kvIter.getValue();
    final int kp = kb.getPosition();
    final int klen = kb.getLength() - kp;
    key.reset(kb.getData(), kp, klen);
    final int vp = vb.getPosition();
    final int vlen = vb.getLength() - vp;
    value.reset(vb.getData(), vp, vlen);
    bytesRead += klen + vlen;
    return true;
  }
  return false;
}
Example 4
Source File: InMemoryWriter.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public void append(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  if (keyLength < 0) {
    throw new IOException("Negative key-length not allowed: " + keyLength + 
                          " for " + key);
  }
  
  int valueLength = value.getLength() - value.getPosition();
  if (valueLength < 0) {
    throw new IOException("Negative value-length not allowed: " + 
                          valueLength + " for " + value);
  }

  WritableUtils.writeVInt(out, keyLength);
  WritableUtils.writeVInt(out, valueLength);
  out.write(key.getData(), key.getPosition(), keyLength); 
  out.write(value.getData(), value.getPosition(), valueLength); 
}
Example 5
Source File: BackupStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Write the key and value to the cache in the IFile format
 * @param key
 * @param value
 * @throws IOException
 */
public void write(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  int valueLength = value.getLength() - value.getPosition();
  WritableUtils.writeVInt(dataOut, keyLength);
  WritableUtils.writeVInt(dataOut, valueLength);
  dataOut.write(key.getData(), key.getPosition(), keyLength);
  dataOut.write(value.getData(), value.getPosition(), valueLength);
  usedSize += keyLength + valueLength + 
    WritableUtils.getVIntSize(keyLength) +
    WritableUtils.getVIntSize(valueLength);
  LOG.debug("ID: " + segmentList.size() + " WRITE TO MEM");
}
Example 6
Source File: MergeManagerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public boolean nextRawKey(DataInputBuffer key) throws IOException {
  if (kvIter.next()) {
    final DataInputBuffer kb = kvIter.getKey();
    final int kp = kb.getPosition();
    final int klen = kb.getLength() - kp;
    key.reset(kb.getData(), kp, klen);
    bytesRead += klen;
    return true;
  }
  return false;
}
Example 7
Source File: MergeManagerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public void nextRawValue(DataInputBuffer value) throws IOException {
  final DataInputBuffer vb = kvIter.getValue();
  final int vp = vb.getPosition();
  final int vlen = vb.getLength() - vp;
  value.reset(vb.getData(), vp, vlen);
  bytesRead += vlen;
}
Example 8
Source File: BackupStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
boolean reserveSpace(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  int valueLength = value.getLength() - value.getPosition();

  int requestedSize = keyLength + valueLength + 
    WritableUtils.getVIntSize(keyLength) +
    WritableUtils.getVIntSize(valueLength);
  return reserveSpace(requestedSize);
}
Example 9
Source File: BufferUtils.java    From tez with Apache License 2.0 5 votes vote down vote up 
public static int compare(DataInputBuffer buf1, DataOutputBuffer buf2) {
  byte[] b1 = buf1.getData();
  byte[] b2 = buf2.getData();
  int s1 = buf1.getPosition();
  int s2 = 0;
  int l1 = buf1.getLength();
  int l2 = buf2.getLength();
  return FastByteComparisons.compareTo(b1, s1, (l1 - s1), b2, s2, l2);
}
Example 10
Source File: Merger.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
protected boolean lessThan(Object a, Object b) {
  DataInputBuffer key1 = ((Segment<K, V>)a).getKey();
  DataInputBuffer key2 = ((Segment<K, V>)b).getKey();
  int s1 = key1.getPosition();
  int l1 = key1.getLength() - s1;
  int s2 = key2.getPosition();
  int l2 = key2.getLength() - s2;

  return comparator.compare(key1.getData(), s1, l1, key2.getData(), s2, l2) < 0;
}
Example 11
Source File: MergeManager.java    From tez with Apache License 2.0 5 votes vote down vote up 
public void nextRawValue(DataInputBuffer value) throws IOException {
  final DataInputBuffer vb = kvIter.getValue();
  final int vp = vb.getPosition();
  final int vlen = vb.getLength() - vp;
  value.reset(vb.getData(), vp, vlen);
  bytesRead += vlen;
}
Example 12
Source File: BackupStore.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Write the key and value to the cache in the IFile format
 * @param key
 * @param value
 * @throws IOException
 */
public void write(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  int valueLength = value.getLength() - value.getPosition();
  WritableUtils.writeVInt(dataOut, keyLength);
  WritableUtils.writeVInt(dataOut, valueLength);
  dataOut.write(key.getData(), key.getPosition(), keyLength);
  dataOut.write(value.getData(), value.getPosition(), valueLength);
  usedSize += keyLength + valueLength + 
    WritableUtils.getVIntSize(keyLength) +
    WritableUtils.getVIntSize(valueLength);
  LOG.debug("ID: " + segmentList.size() + " WRITE TO MEM");
}
Example 13
Source File: PipelinedSorter.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
public void reset(DataInputBuffer clone) {
  byte[] data = clone.getData();
  int start = clone.getPosition();
  int length = clone.getLength();
  resize(length);
  System.arraycopy(data, start, buffer, 0, length);
  super.reset(buffer, 0, length);
}
Example 14
Source File: MergeManagerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public boolean nextRawKey(DataInputBuffer key) throws IOException {
  if (kvIter.next()) {
    final DataInputBuffer kb = kvIter.getKey();
    final int kp = kb.getPosition();
    final int klen = kb.getLength() - kp;
    key.reset(kb.getData(), kp, klen);
    bytesRead += klen;
    return true;
  }
  return false;
}
Example 15
Source File: MergeManagerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public void nextRawValue(DataInputBuffer value) throws IOException {
  final DataInputBuffer vb = kvIter.getValue();
  final int vp = vb.getPosition();
  final int vlen = vb.getLength() - vp;
  value.reset(vb.getData(), vp, vlen);
  bytesRead += vlen;
}
Example 16
Source File: IFile.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Send key/value to be appended to IFile. To represent same key as previous
 * one, send IFile.REPEAT_KEY as key parameter.  Should not call this method with
 * IFile.REPEAT_KEY as the first key.
 *
 * @param key
 * @param value
 * @throws IOException
 */
public void append(DataInputBuffer key, DataInputBuffer value) throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  checkState((key == REPEAT_KEY || keyLength >= 0),
    "Negative key-length not allowed: %d for %s", keyLength, key);

  int valueLength = value.getLength() - value.getPosition();
  checkState(valueLength >= 0,
    "Negative value-length not allowed: %d for %s", valueLength, value);

  boolean sameKey = (key == REPEAT_KEY);
  if (!sameKey && rle) {
    sameKey = (keyLength != 0) && (BufferUtils.compare(previous, key) == 0);
  }

  if (!sameKey) {
    writeKVPair(key.getData(), key.getPosition(), keyLength,
        value.getData(), value.getPosition(), valueLength);
    if (rle) {
      BufferUtils.copy(key, previous);
    }
  } else {
    writeValue(value.getData(), value.getPosition(), valueLength);
  }
  prevKey = (sameKey) ? REPEAT_KEY : key;
  ++numRecordsWritten;
}
Example 17
Source File: MergeManager.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@Override
public KeyState readRawKey(DataInputBuffer key) throws IOException {
  if (kvIter.next()) {
    final DataInputBuffer kb = kvIter.getKey();
    final int kp = kb.getPosition();
    final int klen = kb.getLength() - kp;
    key.reset(kb.getData(), kp, klen);
    bytesRead += klen;
    return KeyState.NEW_KEY;
  }
  return KeyState.NO_KEY;
}
Example 18
Source File: IFile.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Appends the value to previous key. Assumes that the caller has already done relevant checks
 * for identical keys. Also, no validations are done in this method
 *
 * @param value
 * @throws IOException
 */
public void appendValue(DataInputBuffer value) throws IOException {
  int valueLength = value.getLength() - value.getPosition();
  checkState(valueLength >= 0,
      "Negative value-length not allowed: %d for %s", valueLength, value);
  writeValue(value.getData(), value.getPosition(), valueLength);
  buffer.reset();
  ++numRecordsWritten;
  prevKey = REPEAT_KEY;
}
Example 19
Source File: ReduceContextImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Advance to the next key/value pair.
 */
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
  if (!hasMore) {
    key = null;
    value = null;
    return false;
  }
  firstValue = !nextKeyIsSame;
  DataInputBuffer nextKey = input.getKey();
  currentRawKey.set(nextKey.getData(), nextKey.getPosition(), 
                    nextKey.getLength() - nextKey.getPosition());
  buffer.reset(currentRawKey.getBytes(), 0, currentRawKey.getLength());
  key = keyDeserializer.deserialize(key);
  DataInputBuffer nextVal = input.getValue();
  buffer.reset(nextVal.getData(), nextVal.getPosition(), nextVal.getLength()
      - nextVal.getPosition());
  value = valueDeserializer.deserialize(value);

  currentKeyLength = nextKey.getLength() - nextKey.getPosition();
  currentValueLength = nextVal.getLength() - nextVal.getPosition();

  if (isMarked) {
    backupStore.write(nextKey, nextVal);
  }

  hasMore = input.next();
  if (hasMore) {
    nextKey = input.getKey();
    nextKeyIsSame = comparator.compare(currentRawKey.getBytes(), 0, 
                                   currentRawKey.getLength(),
                                   nextKey.getData(),
                                   nextKey.getPosition(),
                                   nextKey.getLength() - nextKey.getPosition()
                                       ) == 0;
  } else {
    nextKeyIsSame = false;
  }
  inputValueCounter.increment(1);
  return true;
}
Example 20
Source File: IFile.java    From tez with Apache License 2.0 3 votes vote down vote up 
/**
 * Appends the value to previous key. Assumes that the caller has already done relevant checks
 * for identical keys. Also, no validations are done in this method. It is caller's responsibility
 * to pass non-negative key/value lengths. Otherwise,IndexOutOfBoundsException could be
 * thrown at runtime.
 *
 * @param value
 * @throws IOException
 */
public void appendValue(DataInputBuffer value) throws IOException {
  int valueLength = value.getLength() - value.getPosition();
  assert(valueLength >= 0);
  writeValue(value.getData(), value.getPosition(), valueLength);
  buffer.reset();
  ++numRecordsWritten;
  prevKey = REPEAT_KEY;
}