Java Code Examples for org.apache.arrow.vector.types.FloatingPointPrecision#DOUBLE
The following examples show how to use
org.apache.arrow.vector.types.FloatingPointPrecision#DOUBLE .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: ArrowUtils.java From konduit-serving with Apache License 2.0 | 6 votes |
|
public static ColumnMetaData metaDataFromField(Field field) {
ArrowType arrowType = field.getFieldType().getType();
if (arrowType instanceof ArrowType.Int) {
ArrowType.Int intType = (ArrowType.Int) arrowType;
return intType.getBitWidth() == 32 ? new IntegerMetaData(field.getName()) : new LongMetaData(field.getName());
} else if (arrowType instanceof ArrowType.Bool) {
return new BooleanMetaData(field.getName());
} else if (arrowType instanceof ArrowType.FloatingPoint) {
ArrowType.FloatingPoint floatingPointType = (ArrowType.FloatingPoint) arrowType;
return floatingPointType.getPrecision() == FloatingPointPrecision.DOUBLE ? new DoubleMetaData(field.getName()) : new FloatMetaData(field.getName());
} else if (arrowType instanceof ArrowType.Binary) {
return new BinaryMetaData(field.getName());
} else if (arrowType instanceof ArrowType.Utf8) {
return new StringMetaData(field.getName());
} else if (arrowType instanceof ArrowType.Date) {
return new TimeMetaData(field.getName());
} else {
throw new IllegalStateException("Illegal type " + field.getFieldType().getType());
}
}
Example 2
| Source File: GlobalDictionaryBuilder.java From dremio-oss with Apache License 2.0 | 5 votes |
|
private static VectorContainer buildDoubleGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) {
final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), null);
final VectorContainer input = new VectorContainer(bufferAllocator);
final Float8Vector doubleVector = input.addOrGet(field);
doubleVector.allocateNew();
SortedSet<Double> values = Sets.newTreeSet();
for (Dictionary dictionary : dictionaries) {
for (int i = 0; i <= dictionary.getMaxId(); ++i) {
values.add(dictionary.decodeToDouble(i));
}
}
if (existingDict != null) {
final Float8Vector existingDictValues = existingDict.getValueAccessorById(Float8Vector.class, 0).getValueVector();
for (int i = 0; i < existingDict.getRecordCount(); ++i) {
values.add(existingDictValues.get(i));
}
}
final Iterator<Double> iter = values.iterator();
int recordCount = 0;
while (iter.hasNext()) {
doubleVector.setSafe(recordCount++, iter.next());
}
doubleVector.setValueCount(recordCount);
input.setRecordCount(recordCount);
input.buildSchema(BatchSchema.SelectionVectorMode.NONE);
return input;
}
Example 3
| Source File: ArrowConverter.java From DataVec with Apache License 2.0 | 5 votes |
|
private static ColumnMetaData metaDataFromField(Field field) {
ArrowType arrowType = field.getFieldType().getType();
if(arrowType instanceof ArrowType.Int) {
val intType = (ArrowType.Int) arrowType;
if(intType.getBitWidth() == 32)
return new IntegerMetaData(field.getName());
else {
return new LongMetaData(field.getName());
}
}
else if(arrowType instanceof ArrowType.Bool) {
return new BooleanMetaData(field.getName());
}
else if(arrowType instanceof ArrowType.FloatingPoint) {
val floatingPointType = (ArrowType.FloatingPoint) arrowType;
if(floatingPointType.getPrecision() == FloatingPointPrecision.DOUBLE)
return new DoubleMetaData(field.getName());
else {
return new FloatMetaData(field.getName());
}
}
else if(arrowType instanceof ArrowType.Binary) {
return new BinaryMetaData(field.getName());
}
else if(arrowType instanceof ArrowType.Utf8) {
return new StringMetaData(field.getName());
}
else if(arrowType instanceof ArrowType.Date) {
return new TimeMetaData(field.getName());
}
else {
throw new IllegalStateException("Illegal type " + field.getFieldType().getType());
}
}
Example 4
| Source File: ArrowConverter.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
private static ColumnMetaData metaDataFromField(Field field) {
ArrowType arrowType = field.getFieldType().getType();
if(arrowType instanceof ArrowType.Int) {
val intType = (ArrowType.Int) arrowType;
if(intType.getBitWidth() == 32)
return new IntegerMetaData(field.getName());
else {
return new LongMetaData(field.getName());
}
}
else if(arrowType instanceof ArrowType.Bool) {
return new BooleanMetaData(field.getName());
}
else if(arrowType instanceof ArrowType.FloatingPoint) {
val floatingPointType = (ArrowType.FloatingPoint) arrowType;
if(floatingPointType.getPrecision() == FloatingPointPrecision.DOUBLE)
return new DoubleMetaData(field.getName());
else {
return new FloatMetaData(field.getName());
}
}
else if(arrowType instanceof ArrowType.Binary) {
return new BinaryMetaData(field.getName());
}
else if(arrowType instanceof ArrowType.Utf8) {
return new StringMetaData(field.getName());
}
else if(arrowType instanceof ArrowType.Date) {
return new TimeMetaData(field.getName());
}
else {
throw new IllegalStateException("Illegal type " + field.getFieldType().getType());
}
}
Example 5
| Source File: SqlTypeNameToArrowType.java From dremio-flight-connector with Apache License 2.0 | 4 votes |
|
public static ArrowType toArrowType(UserProtos.ResultColumnMetadata type) {
String typeName = type.getDataType();
switch (typeName) {
case "NULL":
return new Null();
case "MAP":
return new ArrowType.Map(false); //todo inner type?
case "ARRAY":
return new ArrowType.List(); //todo inner type?
case "UNION":
throw new UnsupportedOperationException("have not implemented unions");
//return new Union(); //todo inner type?
case "TINYINT":
return new Int(8, true);
case "SMALLINT":
return new Int(16, true);
case "INTEGER":
return new Int(32, true);
case "BIGINT":
return new Int(64, true);
case "FLOAT":
return new FloatingPoint(FloatingPointPrecision.SINGLE);
case "DOUBLE":
return new FloatingPoint(FloatingPointPrecision.DOUBLE);
case "CHARACTER VARYING":
return new Utf8();
case "BINARY VARYING":
return new Binary();
case "BOOLEAN":
return new Bool();
case "DECIMAL":
return new Decimal(type.getPrecision(), type.getScale());
case "DATE":
return new Date(DateUnit.MILLISECOND);
case "TIME":
return new Time(TimeUnit.MICROSECOND, 64);
case "TIMESTAMP":
return new Timestamp(TimeUnit.MICROSECOND, "UTC");
case "INTERVAL DAY TO SECOND":
return new Interval(IntervalUnit.DAY_TIME);
case "INTERVAL YEAR TO MONTH":
return new Interval(IntervalUnit.YEAR_MONTH);
case "BINARY":
return new ArrowType.FixedSizeBinary(50);
default:
throw new IllegalStateException("unable to find arrow type for " + typeName);
}
}
Example 6
| Source File: Fixtures.java From dremio-oss with Apache License 2.0 | 4 votes |
|
@Override
ArrowType getType() {
return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
}
Example 7
| Source File: TestDictionaryLookup.java From dremio-oss with Apache License 2.0 | 4 votes |
|
@Test
public void testDictionaryLookup() throws Throwable {
try (final VectorContainer dict1 = new VectorContainer(getTestAllocator());
final VectorContainer dict2 = new VectorContainer(getTestAllocator());
final VectorContainer dict3 = new VectorContainer(getTestAllocator())) {
final Map<String, GlobalDictionaryFieldInfo> dictionaryFieldInfoMap = Maps.newHashMap();
final Field field1 = new Field(SchemaPath.getSimplePath("c0").getAsUnescapedPath(), true, new ArrowType.Int(64, true), null);
final BigIntVector longVector = dict1.addOrGet(field1);
longVector.allocateNew();
longVector.setSafe(0, 10L);
longVector.setSafe(1, 20L);
longVector.setSafe(2, 30L);
longVector.setSafe(3, 40L);
longVector.setSafe(4, 50L);
longVector.setValueCount(5);
dict1.setRecordCount(5);
dict1.buildSchema(BatchSchema.SelectionVectorMode.NONE);
dictionaryFieldInfoMap.put("c0", new GlobalDictionaryFieldInfo(0, "c0", null, field1.getType(), "local"));
final Field field2 = new Field(SchemaPath.getSimplePath("c1").getAsUnescapedPath(), true, new ArrowType.Binary(), null);
final VarBinaryVector binaryVector = dict2.addOrGet(field2);
binaryVector.allocateNew();
binaryVector.setSafe(0, "abc".getBytes(UTF8), 0, 3);
binaryVector.setSafe(1, "bcd".getBytes(UTF8), 0, 3);
binaryVector.setSafe(2, "cde".getBytes(UTF8), 0, 3);
binaryVector.setSafe(3, "def".getBytes(UTF8), 0, 3);
binaryVector.setSafe(4, "efg".getBytes(UTF8), 0, 3);
binaryVector.setValueCount(5);
dict2.setRecordCount(5);
dict2.buildSchema(BatchSchema.SelectionVectorMode.NONE);
dictionaryFieldInfoMap.put("c1", new GlobalDictionaryFieldInfo(0, "c1", null, field2.getType(), "local"));
final Field field3 = new Field(SchemaPath.getSimplePath("c2").getAsUnescapedPath(), true, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), null);
final Float8Vector doubleVector = dict3.addOrGet(field3);
doubleVector.allocateNew();
doubleVector.setSafe(0, 100.1);
doubleVector.setSafe(1, 200.2);
doubleVector.setSafe(2, 300.3);
doubleVector.setSafe(3, 400.4);
doubleVector.setSafe(4, 500.5);
doubleVector.setValueCount(5);
dict3.setRecordCount(5);
dict3.buildSchema(BatchSchema.SelectionVectorMode.NONE);
dictionaryFieldInfoMap.put("c2", new GlobalDictionaryFieldInfo(0, "c2", null, field3.getType(), "local"));
OperatorCreatorRegistry registry = Mockito.mock(OperatorCreatorRegistry.class);
Mockito.when(registry.getSingleInputOperator(Matchers.any(OperatorContext.class), Matchers.any(PhysicalOperator.class)))
.thenAnswer(new Answer<SingleInputOperator>() {
public SingleInputOperator answer(InvocationOnMock invocation) throws Exception {
Object[] args = invocation.getArguments();
DictionaryLookupOperator dictionaryLookupOperator = Mockito.spy(new DictionaryLookupOperator(
(OperatorContext)args[0], (DictionaryLookupPOP)args[1]));
Mockito.doReturn(dict1).when(dictionaryLookupOperator).loadDictionary(Matchers.eq("c0"));
Mockito.doReturn(dict2).when(dictionaryLookupOperator).loadDictionary(Matchers.eq("c1"));
Mockito.doReturn(dict3).when(dictionaryLookupOperator).loadDictionary(Matchers.eq("c2"));
return dictionaryLookupOperator;
}
});
BaseTestOperator.testContext.setRegistry(registry);
DictionaryLookupPOP lookup = new DictionaryLookupPOP(null, PROPS, null, dictionaryFieldInfoMap);
Table input = t(
th("c0", "c1", "c2"),
tr(0, 1, 2),
tr(1, 2, 0),
tr(2, 0, 1)
);
Table output = t(
th("c0", "c1", "c2"),
tr(10L, "bcd".getBytes(UTF8), 300.3),
tr(20L, "cde".getBytes(UTF8), 100.1),
tr(30L, "abc".getBytes(UTF8), 200.2)
);
validateSingle(lookup, DictionaryLookupOperator.class, input, output);
}
}
Example 8
| Source File: HiveSchemaConverter.java From dremio-oss with Apache License 2.0 | 4 votes |
|
public static Field getArrowFieldFromHivePrimitiveType(String name, TypeInfo typeInfo) {
switch (typeInfo.getCategory()) {
case PRIMITIVE:
PrimitiveTypeInfo pTypeInfo = (PrimitiveTypeInfo) typeInfo;
switch (pTypeInfo.getPrimitiveCategory()) {
case BOOLEAN:
return new Field(name, true, new Bool(), null);
case BYTE:
return new Field(name, true, new Int(32, true), null);
case SHORT:
return new Field(name, true, new Int(32, true), null);
case INT:
return new Field(name, true, new Int(32, true), null);
case LONG:
return new Field(name, true, new Int(64, true), null);
case FLOAT:
return new Field(name, true, new FloatingPoint(FloatingPointPrecision.SINGLE), null);
case DOUBLE:
return new Field(name, true, new FloatingPoint(FloatingPointPrecision.DOUBLE), null);
case DATE:
return new Field(name, true, new Date(DateUnit.MILLISECOND), null);
case TIMESTAMP:
return new Field(name, true, new Timestamp(TimeUnit.MILLISECOND, null), null);
case BINARY:
return new Field(name, true, new Binary(), null);
case DECIMAL: {
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) pTypeInfo;
return new Field(name, true, new Decimal(decimalTypeInfo.getPrecision(), decimalTypeInfo.getScale()), null);
}
case STRING:
case VARCHAR:
case CHAR: {
return new Field(name, true, new Utf8(), null);
}
case UNKNOWN:
case VOID:
default:
// fall through.
}
default:
}
return null;
}
Example 9
| Source File: ArrowUtils.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public ArrowType visit(DoubleType doubleType) {
return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
}
