org.apache.arrow.vector.Float8Vector Java Examples

private Pair<Float8Vector, ResultVerifier> testFloat8Vector() {
  final String colName = "colFloat8";
  final List<Double> values = asList(20.2345234d, 503453.023d, -238423.3453453d, 3273452.345324563245d, null);
  Float8Vector valuesVector = new Float8Vector(colName, allocator);
  valuesVector.allocateNew(values.size());
  for (int i = 0; i < values.size(); i++) {
    if (values.get(i) == null) {
      valuesVector.setNull(i);
    } else {
      valuesVector.set(i, values.get(i));
    }
  }

  ResultVerifier verifier = new ResultVerifier() {
    @Override
    public void verify(DataPOJO output) {
      verifyDoubleValues(values, output, colName, 0.01f);
    }
  };

  return Pair.of(valuesVector, verifier);
}
 

@Override
public void doWrite(Row value, int ordinal) {
	if (value.getField(ordinal) == null) {
		((Float8Vector) getValueVector()).setNull(getCount());
	} else {
		((Float8Vector) getValueVector()).setSafe(getCount(), (double) value.getField(ordinal));
	}
}
 

private static VectorContainer buildDoubleGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) {
  final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), null);
  final VectorContainer input = new VectorContainer(bufferAllocator);
  final Float8Vector doubleVector = input.addOrGet(field);
  doubleVector.allocateNew();
  SortedSet<Double> values = Sets.newTreeSet();
  for (Dictionary dictionary : dictionaries) {
    for (int i = 0; i <= dictionary.getMaxId(); ++i) {
      values.add(dictionary.decodeToDouble(i));
    }
  }
  if (existingDict != null) {
    final Float8Vector existingDictValues = existingDict.getValueAccessorById(Float8Vector.class, 0).getValueVector();
    for (int i = 0; i < existingDict.getRecordCount(); ++i) {
      values.add(existingDictValues.get(i));
    }
  }
  final Iterator<Double> iter = values.iterator();
  int recordCount = 0;
  while (iter.hasNext()) {
    doubleVector.setSafe(recordCount++, iter.next());
  }
  doubleVector.setValueCount(recordCount);
  input.setRecordCount(recordCount);
  input.buildSchema(BatchSchema.SelectionVectorMode.NONE);
  return input;
}
 

@Override
public double toDouble(int index)
{
  if (float8Vector.isNull(index))
  {
    return 0;
  }
  else
  {
    return float8Vector.getDataBuffer().getDouble(index * Float8Vector.TYPE_WIDTH);
  }
}
 

@Test
public void T_convert_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  Float8Vector vector = new Float8Vector( "test" , allocator );
  vector.allocateNew();
  vector.setSafe( 0 , (double)0 );  
  vector.setSafe( 1 , (double)1 );  
  vector.setSafe( 2 , (double)0 );  
  vector.setNull( 3 );  
  vector.setSafe( 4 , (double)1 );  
  vector.setSafe( 5 , (double)1 );  
  vector.setSafe( 6 , (double)1 );  
  vector.setNull( 7 );  
  vector.setValueCount( 8 );

  IColumn column = ArrowColumnFactory.convert( "test" , vector );
  assertEquals( column.getColumnName() , "test" );
  assertEquals( column.size() , 8 );
  assertTrue( ( column.getColumnType() == ColumnType.DOUBLE ) );
  assertEquals( ( (PrimitiveObject)( column.get(0).getRow() ) ).getDouble() , (double)0  );
  assertEquals( ( (PrimitiveObject)( column.get(1).getRow() ) ).getDouble() , (double)1  );
  assertEquals( ( (PrimitiveObject)( column.get(2).getRow() ) ).getDouble() , (double)0  );
  assertEquals( column.get(3).getRow() , null  );
  assertEquals( ( (PrimitiveObject)( column.get(4).getRow() ) ).getDouble() , (double)1 );
  assertEquals( ( (PrimitiveObject)( column.get(5).getRow() ) ).getDouble() , (double)1 );
  assertEquals( ( (PrimitiveObject)( column.get(6).getRow() ) ).getDouble() , (double)1 );
  assertEquals( column.get(7).getRow() , null  );
}
 

@NotNull
@SuppressWarnings("checkstyle:CyclomaticComplexity")
private static ArrowVectorAccessor getPlainVectorAccessor(FieldVector vector) {
  if (vector instanceof BitVector) {
    return new BooleanAccessor((BitVector) vector);
  } else if (vector instanceof IntVector) {
    return new IntAccessor((IntVector) vector);
  } else if (vector instanceof BigIntVector) {
    return new LongAccessor((BigIntVector) vector);
  } else if (vector instanceof Float4Vector) {
    return new FloatAccessor((Float4Vector) vector);
  } else if (vector instanceof Float8Vector) {
    return new DoubleAccessor((Float8Vector) vector);
  } else if (vector instanceof IcebergArrowVectors.DecimalArrowVector) {
    return new DecimalAccessor((IcebergArrowVectors.DecimalArrowVector) vector);
  } else if (vector instanceof IcebergArrowVectors.VarcharArrowVector) {
    return new StringAccessor((IcebergArrowVectors.VarcharArrowVector) vector);
  } else if (vector instanceof VarBinaryVector) {
    return new BinaryAccessor((VarBinaryVector) vector);
  } else if (vector instanceof DateDayVector) {
    return new DateAccessor((DateDayVector) vector);
  } else if (vector instanceof TimeStampMicroTZVector) {
    return new TimestampAccessor((TimeStampMicroTZVector) vector);
  } else if (vector instanceof ListVector) {
    ListVector listVector = (ListVector) vector;
    return new ArrayAccessor(listVector);
  } else if (vector instanceof StructVector) {
    StructVector structVector = (StructVector) vector;
    return new StructAccessor(structVector);
  }
  throw new UnsupportedOperationException("Unsupported vector: " + vector.getClass());
}
 

@Test
public void T_convert_1() throws IOException{
  BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
  Float8Vector vector = new Float8Vector( "test" , allocator );
  vector.allocateNew();
  vector.setSafe( 0 , (double)0 );
  vector.setSafe( 1 , (double)1 );
  vector.setSafe( 2 , (double)0 );
  vector.setNull( 3 );
  vector.setSafe( 4 , (double)1 );
  vector.setSafe( 5 , (double)1 );
  vector.setSafe( 6 , (double)1 );
  vector.setNull( 7 );
  vector.setValueCount( 8 );

  IColumn column = ArrowColumnFactory.convert( "test" , vector );
  assertEquals( column.getColumnName() , "test" );
  assertEquals( column.size() , 8 );
  assertTrue( ( column.getColumnType() == ColumnType.DOUBLE ) );
  assertEquals( ( (PrimitiveObject)( column.get(0).getRow() ) ).getDouble() , (double)0  );
  assertEquals( ( (PrimitiveObject)( column.get(1).getRow() ) ).getDouble() , (double)1  );
  assertEquals( ( (PrimitiveObject)( column.get(2).getRow() ) ).getDouble() , (double)0  );
  assertEquals( column.get(3).getRow() , null  );
  assertEquals( ( (PrimitiveObject)( column.get(4).getRow() ) ).getDouble() , (double)1 );
  assertEquals( ( (PrimitiveObject)( column.get(5).getRow() ) ).getDouble() , (double)1 );
  assertEquals( ( (PrimitiveObject)( column.get(6).getRow() ) ).getDouble() , (double)1 );
  assertEquals( column.get(7).getRow() , null  );
}
 

@Override
public void doWrite(T in, int ordinal) {
	if (isNullAt(in, ordinal)) {
		((Float8Vector) getValueVector()).setNull(getCount());
	} else {
		((Float8Vector) getValueVector()).setSafe(getCount(), readDouble(in, ordinal));
	}
}
 

private void populateDouble(Float8Vector vector, Double[] data) {
  vector.allocateNew();
  Random r = new Random();
  for(int i =0; i < data.length; i++){
    Double val = data[i];
    if(val != null){
      vector.setSafe(i, val);
    } else {
      vector.setSafe(i, 0, r.nextDouble());
    }
  }
  vector.setValueCount(data.length);
}
 

private byte[] floatToByteArray(float i)
{
  return ByteBuffer.allocate(Float8Vector.TYPE_WIDTH).putDouble(0, i).array();
}
 

private static ArrowFieldWriter<Row> createRowArrowFieldWriter(ValueVector vector, LogicalType fieldType) {
	if (vector instanceof TinyIntVector) {
		return new RowTinyIntWriter((TinyIntVector) vector);
	} else if (vector instanceof SmallIntVector) {
		return new RowSmallIntWriter((SmallIntVector) vector);
	} else if (vector instanceof IntVector) {
		return new RowIntWriter((IntVector) vector);
	} else if (vector instanceof BigIntVector) {
		return new RowBigIntWriter((BigIntVector) vector);
	} else if (vector instanceof BitVector) {
		return new RowBooleanWriter((BitVector) vector);
	} else if (vector instanceof Float4Vector) {
		return new RowFloatWriter((Float4Vector) vector);
	} else if (vector instanceof Float8Vector) {
		return new RowDoubleWriter((Float8Vector) vector);
	} else if (vector instanceof VarCharVector) {
		return new RowVarCharWriter((VarCharVector) vector);
	} else if (vector instanceof VarBinaryVector) {
		return new RowVarBinaryWriter((VarBinaryVector) vector);
	} else if (vector instanceof DecimalVector) {
		DecimalVector decimalVector = (DecimalVector) vector;
		return new RowDecimalWriter(decimalVector, getPrecision(decimalVector), decimalVector.getScale());
	} else if (vector instanceof DateDayVector) {
		return new RowDateWriter((DateDayVector) vector);
	} else if (vector instanceof TimeSecVector || vector instanceof TimeMilliVector ||
		vector instanceof TimeMicroVector || vector instanceof TimeNanoVector) {
		return new RowTimeWriter(vector);
	} else if (vector instanceof TimeStampVector && ((ArrowType.Timestamp) vector.getField().getType()).getTimezone() == null) {
		return new RowTimestampWriter(vector);
	} else if (vector instanceof ListVector) {
		ListVector listVector = (ListVector) vector;
		LogicalType elementType = ((ArrayType) fieldType).getElementType();
		return new RowArrayWriter(listVector, createRowArrowFieldWriter(listVector.getDataVector(), elementType));
	} else if (vector instanceof StructVector) {
		RowType rowType = (RowType) fieldType;
		ArrowFieldWriter<Row>[] fieldsWriters = new ArrowFieldWriter[rowType.getFieldCount()];
		for (int i = 0; i < fieldsWriters.length; i++) {
			fieldsWriters[i] = createRowArrowFieldWriter(
				((StructVector) vector).getVectorById(i),
				rowType.getTypeAt(i));
		}
		return new RowRowWriter((StructVector) vector, fieldsWriters);
	} else {
		throw new UnsupportedOperationException(String.format(
			"Unsupported type %s.", fieldType));
	}
}
 

@Override
public Double read(int index) {
	return ((Float8Vector) getValueVector()).getObject(index);
}
 

public static DoubleWriter<RowData> forRow(Float8Vector doubleVector) {
	return new DoubleWriterForRow(doubleVector);
}
 

public static DoubleWriter<ArrayData> forArray(Float8Vector doubleVector) {
	return new DoubleWriterForArray(doubleVector);
}
 

private DoubleWriter(Float8Vector doubleVector) {
	super(doubleVector);
}
 

private DoubleWriterForRow(Float8Vector doubleVector) {
	super(doubleVector);
}
 

private DoubleWriterForArray(Float8Vector doubleVector) {
	super(doubleVector);
}
 

public RowDoubleWriter(Float8Vector doubleVector) {
	super(doubleVector);
}
 

public static ColumnVector createColumnVector(ValueVector vector, LogicalType fieldType) {
	if (vector instanceof TinyIntVector) {
		return new ArrowTinyIntColumnVector((TinyIntVector) vector);
	} else if (vector instanceof SmallIntVector) {
		return new ArrowSmallIntColumnVector((SmallIntVector) vector);
	} else if (vector instanceof IntVector) {
		return new ArrowIntColumnVector((IntVector) vector);
	} else if (vector instanceof BigIntVector) {
		return new ArrowBigIntColumnVector((BigIntVector) vector);
	} else if (vector instanceof BitVector) {
		return new ArrowBooleanColumnVector((BitVector) vector);
	} else if (vector instanceof Float4Vector) {
		return new ArrowFloatColumnVector((Float4Vector) vector);
	} else if (vector instanceof Float8Vector) {
		return new ArrowDoubleColumnVector((Float8Vector) vector);
	} else if (vector instanceof VarCharVector) {
		return new ArrowVarCharColumnVector((VarCharVector) vector);
	} else if (vector instanceof VarBinaryVector) {
		return new ArrowVarBinaryColumnVector((VarBinaryVector) vector);
	} else if (vector instanceof DecimalVector) {
		return new ArrowDecimalColumnVector((DecimalVector) vector);
	} else if (vector instanceof DateDayVector) {
		return new ArrowDateColumnVector((DateDayVector) vector);
	} else if (vector instanceof TimeSecVector || vector instanceof TimeMilliVector ||
		vector instanceof TimeMicroVector || vector instanceof TimeNanoVector) {
		return new ArrowTimeColumnVector(vector);
	} else if (vector instanceof TimeStampVector && ((ArrowType.Timestamp) vector.getField().getType()).getTimezone() == null) {
		return new ArrowTimestampColumnVector(vector);
	} else if (vector instanceof ListVector) {
		ListVector listVector = (ListVector) vector;
		return new ArrowArrayColumnVector(listVector,
			createColumnVector(listVector.getDataVector(), ((ArrayType) fieldType).getElementType()));
	} else if (vector instanceof StructVector) {
		StructVector structVector = (StructVector) vector;
		ColumnVector[] fieldColumns = new ColumnVector[structVector.size()];
		for (int i = 0; i < fieldColumns.length; ++i) {
			fieldColumns[i] = createColumnVector(structVector.getVectorById(i), ((RowType) fieldType).getTypeAt(i));
		}
		return new ArrowRowColumnVector(structVector, fieldColumns);
	} else {
		throw new UnsupportedOperationException(String.format(
			"Unsupported type %s.", fieldType));
	}
}
 

@Test
public void testConvertToDouble() throws SFException
{
  final int rowCount = 1000;
  List<Double> expectedValues = new ArrayList<>();
  Set<Integer> nullValIndex = new HashSet<>();
  for (int i = 0; i < rowCount; i++)
  {
    expectedValues.add(random.nextDouble());
  }

  Map<String, String> customFieldMeta = new HashMap<>();
  customFieldMeta.put("logicalType", "REAL");

  FieldType fieldType = new FieldType(true,
                                      Types.MinorType.FLOAT8.getType(),
                                      null, customFieldMeta);

  Float8Vector vector = new Float8Vector("col_one", fieldType, allocator);
  for (int i = 0; i < rowCount; i++)
  {
    boolean isNull = random.nextBoolean();
    if (isNull)
    {
      vector.setNull(i);
      nullValIndex.add(i);
    }
    else
    {
      vector.setSafe(i, expectedValues.get(i));
    }
  }

  ArrowVectorConverter converter = new DoubleToRealConverter(vector, 0, (DataConversionContext) this);

  for (int i = 0; i < rowCount; i++)
  {
    double doubleVal = converter.toDouble(i);
    float floatVal = converter.toFloat(i);
    Object doubleObject = converter.toObject(i);
    String doubleString = converter.toString(i);

    if (nullValIndex.contains(i))
    {
      assertThat(doubleVal, is((double) 0));
      assertThat(floatVal, is((float) 0));
      assertThat(doubleObject, is(nullValue()));
      assertThat(doubleString, is(nullValue()));
      assertThat(false, is(converter.toBoolean(i)));
      assertThat(converter.toBytes(i), is(nullValue()));
    }
    else
    {
      assertThat(doubleVal, is(expectedValues.get(i)));
      assertThat(floatVal, is(expectedValues.get(i).floatValue()));
      assertThat(doubleObject, is(expectedValues.get(i)));
      assertThat(doubleString, is(expectedValues.get(i).toString()));
      final int x = i;
      TestUtil.assertSFException(invalidConversionErrorCode,
                                 () -> converter.toBoolean(x));
      bb = ByteBuffer.wrap(converter.toBytes(i));
      assertThat(doubleVal, is(bb.getDouble()));
    }
  }
  vector.clear();
}
 

DoubleToFloat8Copier(DoubleColumnVector inputVector, Float8Vector outputVector) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
}
 

@Override
public byte[] getBytes(int columnIndex) throws SFException
{
  logger.debug("public byte[] getBytes(int columnIndex)");

  // Column index starts from 1, not 0.
  Object obj = getObjectInternal(columnIndex);
  int columnType = resultSetMetaData.getColumnType(columnIndex);

  if (obj == null)
  {
    return null;
  }

  try
  {
    // For all types except time/date/timestamp data, convert data into byte array. Different methods are needed
    // for different types.
    switch (columnType)
    {
      case Types.FLOAT:
      case Types.DOUBLE:
        return ByteBuffer.allocate(Float8Vector.TYPE_WIDTH).putDouble(0, getDouble(columnIndex)).array();
      case Types.NUMERIC:
      case Types.INTEGER:
      case Types.SMALLINT:
      case Types.TINYINT:
      case Types.BIGINT:
        return getBigDecimal(columnIndex).toBigInteger().toByteArray();
      case Types.VARCHAR:
      case Types.CHAR:
        return getString(columnIndex).getBytes();
      case Types.BOOLEAN:
        return getBoolean(columnIndex) ? new byte[]{1} : new byte[]{0};
      case Types.TIMESTAMP:
      case Types.TIME:
      case Types.DATE:
      case Types.DECIMAL:
        throw new SFException(ErrorCode.INVALID_VALUE_CONVERT, columnType, SnowflakeUtil.BYTES_STR,
                              getObjectInternal(columnIndex));
      default:
        return SFBinary.fromHex(obj.toString()).getBytes();
    }
  }
  catch (IllegalArgumentException ex)
  {
    throw new SFException(ErrorCode.INVALID_VALUE_CONVERT, columnType, SnowflakeUtil.BYTES_STR,
                          getObjectInternal(columnIndex));
  }
}
 

public static SqlAccessor getSqlAccessor(ValueVector vector, TimeZone defaultTz) {
  final MinorType type = org.apache.arrow.vector.types.Types.getMinorTypeForArrowType(vector.getField().getType());
  switch(type){
  case UNION:
    return new UnionSqlAccessor((UnionVector) vector);
  case TINYINT:
    return new TinyIntAccessor((TinyIntVector) vector);
  case UINT1:
    return new UInt1Accessor((UInt1Vector) vector);
  case UINT2:
    return new UInt2Accessor((UInt2Vector) vector);
  case SMALLINT:
    return new SmallIntAccessor((SmallIntVector) vector);
  case INT:
    return new IntAccessor((IntVector) vector);
  case UINT4:
    return new UInt4Accessor((UInt4Vector) vector);
  case FLOAT4:
    return new Float4Accessor((Float4Vector) vector);
  case INTERVALYEAR:
    return new IntervalYearAccessor((IntervalYearVector) vector);
  case TIMEMILLI:
    return new TimeMilliAccessor((TimeMilliVector) vector, defaultTz);
  case BIGINT:
    return new BigIntAccessor((BigIntVector) vector);
  case UINT8:
    return new UInt8Accessor((UInt8Vector) vector);
  case FLOAT8:
    return new Float8Accessor((Float8Vector) vector);
  case DATEMILLI:
    return new DateMilliAccessor((DateMilliVector) vector, defaultTz);
  case TIMESTAMPMILLI:
    return new TimeStampMilliAccessor((TimeStampMilliVector) vector, defaultTz);
  case INTERVALDAY:
    return new IntervalDayAccessor((IntervalDayVector) vector);
  case DECIMAL:
    return new DecimalAccessor((DecimalVector) vector);
  case FIXEDSIZEBINARY:
    return new FixedSizeBinaryAccessor((FixedSizeBinaryVector) vector);
  case VARBINARY:
    return new VarBinaryAccessor((VarBinaryVector) vector);
  case VARCHAR:
    return new VarCharAccessor((VarCharVector) vector);
  case BIT:
    return new BitAccessor((BitVector) vector);
  case STRUCT:
  case LIST:
    return new GenericAccessor(vector);
  }
  throw new UnsupportedOperationException(String.format("Unable to find sql accessor for minor type [%s]", type));
}
 

@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final double value = (double) ((DoubleObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
  ((Float8Vector) outputVV).setSafe(outputIndex, value);
}
 

public DoubleToRealConverter(ValueVector fieldVector, int columnIndex, DataConversionContext context)
{
  super(SnowflakeType.REAL.name(), fieldVector, columnIndex, context);
  this.float8Vector = (Float8Vector) fieldVector;
}
 

@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final double value = (double) ((DoubleObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
  ((Float8Vector) outputVV).setSafe(outputIndex, value);
}
 

DoubleToFloat8Copier(DoubleColumnVector inputVector, Float8Vector outputVector) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
}
 

LongToFloat8Copier(LongColumnVector inputVector, Float8Vector outputVector) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
}
 

BytesToFloat8Copier(BytesColumnVector inputVector, Float8Vector outputVector) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
}
 

@Test
public void testUDF() throws Throwable {
  int numRecords = 0;
  String planString = Resources.toString(Resources.getResource("functions/hive/UDF.json"), Charsets.UTF_8);
  List<QueryDataBatch> results = testPhysicalWithResults(planString);

  RecordBatchLoader batchLoader = new RecordBatchLoader(getAllocator());
  for (QueryDataBatch result : results) {
    batchLoader.load(result.getHeader().getDef(), result.getData());
    if (batchLoader.getRecordCount() <= 0) {
      result.release();
      batchLoader.clear();
      continue;
    }

    // Output columns and types
    // 1. str1 : VarChar
    // 2. str1Length : Int
    // 3. str1Ascii : Int
    // 4. flt1 : Float4
    // 5. pow : Float8
    VarCharVector str1V = (VarCharVector) batchLoader.getValueAccessorById(VarCharVector.class, 0).getValueVector();
    IntVector str1LengthV = (IntVector) batchLoader.getValueAccessorById(IntVector.class, 1).getValueVector();
    IntVector str1AsciiV = (IntVector) batchLoader.getValueAccessorById(IntVector.class, 2).getValueVector();
    Float4Vector flt1V = (Float4Vector) batchLoader.getValueAccessorById(Float4Vector.class, 3).getValueVector();
    Float8Vector powV = (Float8Vector) batchLoader.getValueAccessorById(Float8Vector.class, 4).getValueVector();

    for (int i=0; i<batchLoader.getRecordCount(); i++) {
      if (str1V.isNull(i)) {
        continue;
      }
      String str1 = new String(str1V.get(i), Charsets.UTF_8);
      long str1Length = str1LengthV.get(i);
      assertTrue(str1.length() == str1Length);

      int str1Ascii = str1AsciiV.get(i);

      float flt1 = flt1V.get(i);

      double pow = 0;
      if (!powV.isNull(i)) {
        pow = powV.get(i);
        assertTrue(Math.pow(flt1, 2.0) == pow);
      }
      numRecords++;
    }

    result.release();
    batchLoader.clear();
  }
}