Java Code Examples for org.apache.spark.unsafe.Platform#BYTE_ARRAY_OFFSET

@Override
public UnsafeRow next() {
  if (current == null || nextRow >= current.size) {
    current = reader.next();
    nextRow = 0;
  }
  // Reset the holder to start the buffer over again.
  // BufferHolder.reset does the wrong thing...
  holder.cursor = Platform.BYTE_ARRAY_OFFSET;
  writer.reset();
  for(int c=0; c < current.cols.length; ++c) {
    converter[c].convert(writer, c, current.cols[c], nextRow);
  }
  nextRow++;
  return row;
}
 

/**
 * Write this UnsafeRow's underlying bytes to the given OutputStream.
 *
 * @param out         the stream to write to.
 * @param writeBuffer a byte array for buffering chunks of off-heap data while writing to the
 *                    output stream. If this row is backed by an on-heap byte array, then this
 *                    buffer will not be used and may be null.
 */
public void writeToStream(OutputStream out, byte[] writeBuffer) throws IOException {
    if (baseObject instanceof byte[]) {
        int offsetInByteArray = (int) (Platform.BYTE_ARRAY_OFFSET - baseOffset);
        out.write((byte[]) baseObject, offsetInByteArray, sizeInBytes);
    } else {
        int dataRemaining = sizeInBytes;
        long rowReadPosition = baseOffset;
        while (dataRemaining > 0) {
            int toTransfer = Math.min(writeBuffer.length, dataRemaining);
            Platform.copyMemory(
                    baseObject, rowReadPosition, writeBuffer, Platform.BYTE_ARRAY_OFFSET, toTransfer);
            out.write(writeBuffer, 0, toTransfer);
            rowReadPosition += toTransfer;
            dataRemaining -= toTransfer;
        }
    }
}
 

private void checkInts(DataPack[] dataPacks, int entryCount, ByteBufferReader entryFileReader, ByteBuffer bitmapFileBuffer) throws IOException {
    int packCount = dataPacks.length;
    int bitmapSize = DirectBitMap.bits2words(packCount) << 3;
    byte[] bitmapBuffer = new byte[bitmapSize];
    DirectBitMap bitMap = new DirectBitMap(bitmapBuffer, Platform.BYTE_ARRAY_OFFSET, bitmapSize >>> 3, null);

    for (int packId = 0; packId < packCount; packId++) {
        DataPack dataPack = dataPacks[packId];
        for (int rowId = 0; rowId < dataPack.valueCount(); rowId++) {
            int value = dataPack.intValueAt(rowId);
            int pos = BinarySearch.binarySearchInts(entryFileReader, entryCount, value);
            Assert.assertTrue(pos >= 0);
            bitmapFileBuffer.position(pos * bitmapSize);
            bitmapFileBuffer.get(bitmapBuffer);
            Assert.assertTrue(bitMap.get(packId));
        }
    }
}
 

private void checkLongs(DataPack[] dataPacks, int entryCount, ByteBufferReader entryFileReader, ByteBuffer bitmapFileBuffer) throws IOException {
    int packCount = dataPacks.length;
    int bitmapSize = DirectBitMap.bits2words(packCount) << 3;
    byte[] bitmapBuffer = new byte[bitmapSize];
    DirectBitMap bitMap = new DirectBitMap(bitmapBuffer, Platform.BYTE_ARRAY_OFFSET, bitmapSize >>> 3, null);

    for (int packId = 0; packId < packCount; packId++) {
        DataPack dataPack = dataPacks[packId];
        for (int rowId = 0; rowId < dataPack.valueCount(); rowId++) {
            long value = dataPack.longValueAt(rowId);
            int pos = BinarySearch.binarySearchLongs(entryFileReader, entryCount, value);
            Assert.assertTrue(pos >= 0);
            bitmapFileBuffer.position(pos * bitmapSize);
            bitmapFileBuffer.get(bitmapBuffer);
            Assert.assertTrue(bitMap.get(packId));
        }
    }
}
 

private void checkFloats(DataPack[] dataPacks, int entryCount, ByteBufferReader entryFileReader, ByteBuffer bitmapFileBuffer) throws IOException {
    int packCount = dataPacks.length;
    int bitmapSize = DirectBitMap.bits2words(packCount) << 3;
    byte[] bitmapBuffer = new byte[bitmapSize];
    DirectBitMap bitMap = new DirectBitMap(bitmapBuffer, Platform.BYTE_ARRAY_OFFSET, bitmapSize >>> 3, null);

    for (int packId = 0; packId < packCount; packId++) {
        DataPack dataPack = dataPacks[packId];
        for (int rowId = 0; rowId < dataPack.valueCount(); rowId++) {
            float value = dataPack.floatValueAt(rowId);
            int pos = BinarySearch.binarySearchFloats(entryFileReader, entryCount, value);
            Assert.assertTrue(pos >= 0);
            bitmapFileBuffer.position(pos * bitmapSize);
            bitmapFileBuffer.get(bitmapBuffer);
            Assert.assertTrue(bitMap.get(packId));
        }
    }
}
 

private void checkDoubles(DataPack[] dataPacks, int entryCount, ByteBufferReader entryFileReader, ByteBuffer bitmapFileBuffer) throws IOException {
    int packCount = dataPacks.length;
    int bitmapSize = DirectBitMap.bits2words(packCount) << 3;
    byte[] bitmapBuffer = new byte[bitmapSize];
    DirectBitMap bitMap = new DirectBitMap(bitmapBuffer, Platform.BYTE_ARRAY_OFFSET, bitmapSize >>> 3, null);

    for (int packId = 0; packId < packCount; packId++) {
        DataPack dataPack = dataPacks[packId];
        for (int rowId = 0; rowId < dataPack.valueCount(); rowId++) {
            double value = dataPack.doubleValueAt(rowId);
            int pos = BinarySearch.binarySearchDoubles(entryFileReader, entryCount, value);
            Assert.assertTrue(pos >= 0);
            bitmapFileBuffer.position(pos * bitmapSize);
            bitmapFileBuffer.get(bitmapBuffer);
            Assert.assertTrue(bitMap.get(packId));
        }
    }
}
 

private void checkStrings(DataPack[] dataPacks, int entryCount, ByteBufferReader entryFileReader, ByteBuffer bitmapFileBuffer) throws IOException {
    int packCount = dataPacks.length;
    int bitmapSize = DirectBitMap.bits2words(packCount) << 3;
    byte[] bitmapBuffer = new byte[bitmapSize];
    DirectBitMap bitMap = new DirectBitMap(bitmapBuffer, Platform.BYTE_ARRAY_OFFSET, bitmapSize >>> 3, null);

    StringsStructOnByteBufferReader stringsStruct = new StringsStructOnByteBufferReader(entryCount, entryFileReader);
    byte[] keyBuffer = new byte[ColumnType.MAX_STRING_UTF8_SIZE];
    BytePiece value = new BytePiece();
    for (int packId = 0; packId < packCount; packId++) {
        DataPack dataPack = dataPacks[packId];
        for (int rowId = 0; rowId < dataPack.valueCount(); rowId++) {
            dataPack.rawValueAt(rowId, value);
            int pos = BinarySearch.binarySearchStrings(stringsStruct, value.base, value.addr, value.len, keyBuffer);
            Assert.assertTrue(pos >= 0);
            bitmapFileBuffer.position(pos * bitmapSize);
            bitmapFileBuffer.get(bitmapBuffer);
            Assert.assertTrue(bitMap.get(packId));
        }
    }
}
 

/**
 * Returns the underlying bytes for this UnsafeRow.
 */
public byte[] getBytes() {
    if (baseObject instanceof byte[] && baseOffset == Platform.BYTE_ARRAY_OFFSET
            && (((byte[]) baseObject).length == sizeInBytes)) {
        return (byte[]) baseObject;
    } else {
        byte[] bytes = new byte[sizeInBytes];
        Platform.copyMemory(baseObject, baseOffset, bytes, Platform.BYTE_ARRAY_OFFSET, sizeInBytes);
        return bytes;
    }
}
 

@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    this.baseOffset = Platform.BYTE_ARRAY_OFFSET;
    this.sizeInBytes = in.readInt();
    this.numFields = in.readInt();
    this.baseObject = new byte[sizeInBytes];
    in.readFully((byte[]) baseObject);
}