Java Code Examples for org.apache.arrow.vector.DecimalVector#TYPE_WIDTH

The following examples show how to use org.apache.arrow.vector.DecimalVector#TYPE_WIDTH . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: VectorizedParquetDefinitionLevelReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
public void readBatchOfIntLongBackedDecimals(
    final FieldVector vector, final int startOffset,
    final int typeWidth, final int numValsToRead, NullabilityHolder nullabilityHolder,
    ValuesAsBytesReader valuesReader) {
  int bufferIdx = startOffset;
  int left = numValsToRead;
  while (left > 0) {
    if (this.currentCount == 0) {
      this.readNextGroup();
    }
    int num = Math.min(left, this.currentCount);
    byte[] byteArray = new byte[DecimalVector.TYPE_WIDTH];
    switch (mode) {
      case RLE:
        if (currentValue == maxDefLevel) {
          for (int i = 0; i < num; i++) {
            setIntLongBackedDecimal(vector, typeWidth, nullabilityHolder, valuesReader, bufferIdx, byteArray);
            bufferIdx++;
          }
        } else {
          setNulls(nullabilityHolder, bufferIdx, num, vector.getValidityBuffer());
          bufferIdx += num;
        }
        break;
      case PACKED:
        for (int i = 0; i < num; ++i) {
          if (packedValuesBuffer[packedValuesBufferIdx++] == maxDefLevel) {
            setIntLongBackedDecimal(vector, typeWidth, nullabilityHolder, valuesReader, bufferIdx, byteArray);
          } else {
            setNull(nullabilityHolder, bufferIdx, vector.getValidityBuffer());
          }
          bufferIdx++;
        }
        break;
    }
    left -= num;
    currentCount -= num;
  }
}
Example 2
Source File: Dremio09BackwardCompatibilityHandler.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
/**
 * swap the bytes in place to get the BE byte order in NullableDecimalVector
 * @param dataBuffer data buffer of decimal vector
 */
static void patchDecimal(final NettyArrowBuf dataBuffer) {
  final int decimalLength = DecimalVector.TYPE_WIDTH;
  int startPoint = dataBuffer.readerIndex();
  final int valueCount = dataBuffer.readableBytes()/decimalLength;
  for (int i = 0; i < valueCount; i++) {
    for (int j = startPoint, k = startPoint + decimalLength - 1; j < k; j++, k--) {
      final byte firstByte = dataBuffer.getByte(j);
      final byte lastByte = dataBuffer.getByte(k);
      dataBuffer.setByte(j, lastByte);
      dataBuffer.setByte(k, firstByte);
    }
    startPoint += decimalLength;
  }
}
Example 3
Source File: DrillBackwardsCompatibilityHandler.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
/**
 * Dremio has 16 byte little endian decimal format. When sending data to Drill clients
 * we map our data to 24 byte DECIMAL38SPARSE format in Drill.
 * @param dataBuffer data buffer of decimal vector
 */
static ByteBuf patchDecimal(BufferAllocator allocator, final NettyArrowBuf dataBuffer,
                            final SerializedField.Builder decimalField, final SerializedField.Builder childDecimalField) {
  final int decimalLength = DecimalVector.TYPE_WIDTH;
  final int startPoint = dataBuffer.readerIndex();
  final int valueCount = dataBuffer.readableBytes()/decimalLength;
  final ByteBuf drillBuffer = allocator.buffer(dataBuffer.readableBytes() + 8*valueCount).asNettyBuffer();
  int length = 0;
  for (int i = startPoint; i < startPoint + dataBuffer.readableBytes() - 1; i+=decimalLength) {
    final BigDecimal arrowDecimal = DecimalUtility.getBigDecimalFromArrowBuf(dataBuffer.arrowBuf(), i/16,
      decimalField.getMajorType().getScale());
    final int startIndex = (i == startPoint) ? i : i + length;
    DecimalHelper.getSparseFromBigDecimal(arrowDecimal, drillBuffer, startIndex, decimalField.getMajorType().getScale(), NUMBER_DECIMAL_DIGITS);
    length += 8;
  }

  TypeProtos.MajorType.Builder majorTypeBuilder = TypeProtos.MajorType.newBuilder()
    .setMinorType(MinorType.DECIMAL38SPARSE)
    .setMode(DataMode.OPTIONAL)
    .setPrecision(decimalField.getMajorType().getPrecision())
    .setScale(decimalField.getMajorType().getScale());

  decimalField.setMajorType(majorTypeBuilder.build());
  decimalField.setBufferLength(decimalField.getBufferLength() + 8*valueCount);
  drillBuffer.writerIndex(dataBuffer.readableBytes() + 8*valueCount);
  childDecimalField.setMajorType(com.dremio.common.types.Types.required(MinorType.DECIMAL38SPARSE));
  childDecimalField.setBufferLength(childDecimalField.getBufferLength() + 8*valueCount);
  dataBuffer.release();

  return drillBuffer;
}
Example 4
Source File: VectorizedParquetDefinitionLevelReader.java    From iceberg with Apache License 2.0 4 votes vote down vote up 
public void readBatchOfFixedLengthDecimals(
    final FieldVector vector, final int startOffset,
    final int typeWidth, final int numValsToRead, NullabilityHolder nullabilityHolder,
    ValuesAsBytesReader valuesReader) {
  int bufferIdx = startOffset;
  int left = numValsToRead;
  while (left > 0) {
    if (this.currentCount == 0) {
      this.readNextGroup();
    }
    int num = Math.min(left, this.currentCount);
    byte[] byteArray = new byte[DecimalVector.TYPE_WIDTH];
    switch (mode) {
      case RLE:
        if (currentValue == maxDefLevel) {
          for (int i = 0; i < num; i++) {
            valuesReader.getBuffer(typeWidth).get(byteArray, DecimalVector.TYPE_WIDTH - typeWidth, typeWidth);
            ((DecimalVector) vector).setBigEndian(bufferIdx, byteArray);
            nullabilityHolder.setNotNull(bufferIdx);
            bufferIdx++;
          }
        } else {
          setNulls(nullabilityHolder, bufferIdx, num, vector.getValidityBuffer());
          bufferIdx += num;
        }
        break;
      case PACKED:
        for (int i = 0; i < num; ++i) {
          if (packedValuesBuffer[packedValuesBufferIdx++] == maxDefLevel) {
            valuesReader.getBuffer(typeWidth).get(byteArray, DecimalVector.TYPE_WIDTH - typeWidth, typeWidth);
            ((DecimalVector) vector).setBigEndian(bufferIdx, byteArray);
            nullabilityHolder.setNotNull(bufferIdx);
          } else {
            setNull(nullabilityHolder, bufferIdx, vector.getValidityBuffer());
          }
          bufferIdx++;
        }
        break;
    }
    left -= num;
    currentCount -= num;
  }
}
Example 5
Source File: VectorizedParquetDefinitionLevelReader.java    From iceberg with Apache License 2.0 4 votes vote down vote up 
public void readBatchOfDictionaryEncodedFixedLengthDecimals(
    final FieldVector vector,
    final int startOffset,
    final int typeWidth,
    final int numValsToRead,
    NullabilityHolder nullabilityHolder,
    VectorizedDictionaryEncodedParquetValuesReader dictionaryEncodedValuesReader,
    Dictionary dict) {
  int idx = startOffset;
  int left = numValsToRead;
  while (left > 0) {
    if (this.currentCount == 0) {
      this.readNextGroup();
    }
    int num = Math.min(left, this.currentCount);
    switch (mode) {
      case RLE:
        if (currentValue == maxDefLevel) {
          dictionaryEncodedValuesReader.readBatchOfDictionaryEncodedFixedLengthDecimals(vector, typeWidth, idx,
              num, dict, nullabilityHolder);
        } else {
          setNulls(nullabilityHolder, idx, num, vector.getValidityBuffer());
        }
        idx += num;
        break;
      case PACKED:
        for (int i = 0; i < num; i++) {
          if (packedValuesBuffer[packedValuesBufferIdx++] == maxDefLevel) {
            ByteBuffer decimalBytes = dict.decodeToBinary(dictionaryEncodedValuesReader.readInteger()).toByteBuffer();
            byte[] vectorBytes = new byte[DecimalVector.TYPE_WIDTH];
            System.arraycopy(decimalBytes, 0, vectorBytes, DecimalVector.TYPE_WIDTH - typeWidth, typeWidth);
            ((DecimalVector) vector).setBigEndian(idx, vectorBytes);
            nullabilityHolder.setNotNull(idx);
          } else {
            setNull(nullabilityHolder, idx, vector.getValidityBuffer());
          }
          idx++;
        }
        break;
    }
    left -= num;
    currentCount -= num;
  }
}
Example 6
Source File: TestBackwardsCompatibilityHandler.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
@Test
public void testPatchDecimal() {
  DecimalVector decimalVector = new DecimalVector("decimal", allocator, 38, 9);
  decimalVector.allocateNew(8);

  BigDecimal decimal1 = new BigDecimal("123456789.000000000");
  BigDecimal decimal2 = new BigDecimal("11.123456789");
  BigDecimal decimal3 = new BigDecimal("1.000000000");
  BigDecimal decimal4 = new BigDecimal("0.111111111");
  BigDecimal decimal5 = new BigDecimal("-987654321.123456789");
  BigDecimal decimal6 = new BigDecimal("-222222222222.222222222");
  BigDecimal decimal7 = new BigDecimal("-7777777777777.666666667");
  BigDecimal decimal8 = new BigDecimal("1212121212.343434343");

  decimalVector.set(0, decimal1);
  decimalVector.set(1, decimal2);
  decimalVector.set(2, decimal3);
  decimalVector.set(3, decimal4);
  decimalVector.set(4, decimal5);
  decimalVector.set(5, decimal6);
  decimalVector.set(6, decimal7);
  decimalVector.set(7, decimal8);

  decimalVector.setValueCount(8);

  assertEquals(8, decimalVector.getValueCount());
  assertEquals(decimal1, decimalVector.getObject(0));
  assertEquals(decimal2, decimalVector.getObject(1));
  assertEquals(decimal3, decimalVector.getObject(2));
  assertEquals(decimal4, decimalVector.getObject(3));
  assertEquals(decimal5, decimalVector.getObject(4));
  assertEquals(decimal6, decimalVector.getObject(5));
  assertEquals(decimal7, decimalVector.getObject(6));
  assertEquals(decimal8, decimalVector.getObject(7));

  FixedWidthVectorHelper vectorHelper = new FixedWidthVectorHelper(decimalVector);
  SerializedField.Builder decimalField = vectorHelper.getMetadataBuilder();
  SerializedField.Builder childDecimalField = decimalField.getChildBuilderList().get(1);
  ByteBuf newBuffer = patchDecimal(allocator, decimalVector.getDataBuffer().asNettyBuffer(), decimalField,
    childDecimalField);

  int startIndex = 0;
  BigDecimal bd = DecimalHelper.getBigDecimalFromSparse(newBuffer, startIndex, DrillBackwardsCompatibilityHandler.NUMBER_DECIMAL_DIGITS, decimalVector.getScale());
  assertEquals(bd, decimal1);
  startIndex += DecimalVector.TYPE_WIDTH + 8;

  bd = DecimalHelper.getBigDecimalFromSparse(newBuffer, startIndex, DrillBackwardsCompatibilityHandler.NUMBER_DECIMAL_DIGITS, decimalVector.getScale());
  assertEquals(bd, decimal2);
  startIndex += DecimalVector.TYPE_WIDTH + 8;

  bd = DecimalHelper.getBigDecimalFromSparse(newBuffer, startIndex, DrillBackwardsCompatibilityHandler.NUMBER_DECIMAL_DIGITS, decimalVector.getScale());
  assertEquals(bd, decimal3);
  startIndex += DecimalVector.TYPE_WIDTH + 8;

  bd = DecimalHelper.getBigDecimalFromSparse(newBuffer, startIndex, DrillBackwardsCompatibilityHandler.NUMBER_DECIMAL_DIGITS, decimalVector.getScale());
  assertEquals(bd, decimal4);
  startIndex += DecimalVector.TYPE_WIDTH + 8;

  bd = DecimalHelper.getBigDecimalFromSparse(newBuffer, startIndex, DrillBackwardsCompatibilityHandler.NUMBER_DECIMAL_DIGITS, decimalVector.getScale());
  assertEquals(bd, decimal5);
  startIndex += DecimalVector.TYPE_WIDTH + 8;

  bd = DecimalHelper.getBigDecimalFromSparse(newBuffer, startIndex, DrillBackwardsCompatibilityHandler.NUMBER_DECIMAL_DIGITS, decimalVector.getScale());
  assertEquals(bd, decimal6);
  startIndex += DecimalVector.TYPE_WIDTH + 8;

  bd = DecimalHelper.getBigDecimalFromSparse(newBuffer, startIndex, DrillBackwardsCompatibilityHandler.NUMBER_DECIMAL_DIGITS, decimalVector.getScale());
  assertEquals(bd, decimal7);
  startIndex += DecimalVector.TYPE_WIDTH + 8;

  bd = DecimalHelper.getBigDecimalFromSparse(newBuffer, startIndex, DrillBackwardsCompatibilityHandler.NUMBER_DECIMAL_DIGITS, decimalVector.getScale());
  assertEquals(bd, decimal8);

  final ArrowBuf validityBuffer = decimalVector.getValidityBuffer();
  validityBuffer.release();
  newBuffer.release();
}