Java Code Examples for org.apache.arrow.vector.types.pojo.ArrowType#Binary
The following examples show how to use
org.apache.arrow.vector.types.pojo.ArrowType#Binary .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: ArrowUtils.java From konduit-serving with Apache License 2.0 | 6 votes |
|
public static ColumnMetaData metaDataFromField(Field field) {
ArrowType arrowType = field.getFieldType().getType();
if (arrowType instanceof ArrowType.Int) {
ArrowType.Int intType = (ArrowType.Int) arrowType;
return intType.getBitWidth() == 32 ? new IntegerMetaData(field.getName()) : new LongMetaData(field.getName());
} else if (arrowType instanceof ArrowType.Bool) {
return new BooleanMetaData(field.getName());
} else if (arrowType instanceof ArrowType.FloatingPoint) {
ArrowType.FloatingPoint floatingPointType = (ArrowType.FloatingPoint) arrowType;
return floatingPointType.getPrecision() == FloatingPointPrecision.DOUBLE ? new DoubleMetaData(field.getName()) : new FloatMetaData(field.getName());
} else if (arrowType instanceof ArrowType.Binary) {
return new BinaryMetaData(field.getName());
} else if (arrowType instanceof ArrowType.Utf8) {
return new StringMetaData(field.getName());
} else if (arrowType instanceof ArrowType.Date) {
return new TimeMetaData(field.getName());
} else {
throw new IllegalStateException("Illegal type " + field.getFieldType().getType());
}
}
Example 2
| Source File: GlobalDictionaryBuilder.java From dremio-oss with Apache License 2.0 | 5 votes |
|
private static VectorContainer buildBinaryGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) {
final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.Binary(), null);
final VectorContainer input = new VectorContainer(bufferAllocator);
final VarBinaryVector binaryVector = input.addOrGet(field);
binaryVector.allocateNew();
final SortedSet<Binary> values = new TreeSet<>();
for (Dictionary dictionary : dictionaries) {
for (int i = 0; i <= dictionary.getMaxId(); ++i) {
values.add(dictionary.decodeToBinary(i));
}
}
if (existingDict != null) {
final VarBinaryVector existingDictValues = existingDict.getValueAccessorById(VarBinaryVector.class, 0).getValueVector();
for (int i = 0; i < existingDict.getRecordCount(); ++i) {
values.add(Binary.fromConstantByteArray(existingDictValues.get(i)));
}
}
final Iterator<Binary> iter = values.iterator();
int recordCount = 0;
while (iter.hasNext()) {
final byte[] data = iter.next().getBytes();
binaryVector.setSafe(recordCount++, data, 0, data.length);
}
binaryVector.setValueCount(recordCount);
input.setRecordCount(recordCount);
input.buildSchema(BatchSchema.SelectionVectorMode.NONE);
return input;
}
Example 3
| Source File: ArrowConverter.java From DataVec with Apache License 2.0 | 5 votes |
|
private static ColumnMetaData metaDataFromField(Field field) {
ArrowType arrowType = field.getFieldType().getType();
if(arrowType instanceof ArrowType.Int) {
val intType = (ArrowType.Int) arrowType;
if(intType.getBitWidth() == 32)
return new IntegerMetaData(field.getName());
else {
return new LongMetaData(field.getName());
}
}
else if(arrowType instanceof ArrowType.Bool) {
return new BooleanMetaData(field.getName());
}
else if(arrowType instanceof ArrowType.FloatingPoint) {
val floatingPointType = (ArrowType.FloatingPoint) arrowType;
if(floatingPointType.getPrecision() == FloatingPointPrecision.DOUBLE)
return new DoubleMetaData(field.getName());
else {
return new FloatMetaData(field.getName());
}
}
else if(arrowType instanceof ArrowType.Binary) {
return new BinaryMetaData(field.getName());
}
else if(arrowType instanceof ArrowType.Utf8) {
return new StringMetaData(field.getName());
}
else if(arrowType instanceof ArrowType.Date) {
return new TimeMetaData(field.getName());
}
else {
throw new IllegalStateException("Illegal type " + field.getFieldType().getType());
}
}
Example 4
| Source File: ArrowConverter.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
private static ColumnMetaData metaDataFromField(Field field) {
ArrowType arrowType = field.getFieldType().getType();
if(arrowType instanceof ArrowType.Int) {
val intType = (ArrowType.Int) arrowType;
if(intType.getBitWidth() == 32)
return new IntegerMetaData(field.getName());
else {
return new LongMetaData(field.getName());
}
}
else if(arrowType instanceof ArrowType.Bool) {
return new BooleanMetaData(field.getName());
}
else if(arrowType instanceof ArrowType.FloatingPoint) {
val floatingPointType = (ArrowType.FloatingPoint) arrowType;
if(floatingPointType.getPrecision() == FloatingPointPrecision.DOUBLE)
return new DoubleMetaData(field.getName());
else {
return new FloatMetaData(field.getName());
}
}
else if(arrowType instanceof ArrowType.Binary) {
return new BinaryMetaData(field.getName());
}
else if(arrowType instanceof ArrowType.Utf8) {
return new StringMetaData(field.getName());
}
else if(arrowType instanceof ArrowType.Date) {
return new TimeMetaData(field.getName());
}
else {
throw new IllegalStateException("Illegal type " + field.getFieldType().getType());
}
}
Example 5
| Source File: ArrowTypeSerDe.java From aws-athena-query-federation with Apache License 2.0 | 4 votes |
|
private Serializer()
{
super(ArrowType.class, ArrowType.Binary.class);
}
Example 6
| Source File: ArrowTypeSerDe.java From aws-athena-query-federation with Apache License 2.0 | 4 votes |
|
private Deserializer()
{
super(ArrowType.class, ArrowType.Binary.class);
}
Example 7
| Source File: ArrowTypeSerDe.java From aws-athena-query-federation with Apache License 2.0 | 4 votes |
|
@Override
protected ArrowType doTypedDeserialize(JsonParser jparser, DeserializationContext ctxt)
throws IOException
{
return new ArrowType.Binary();
}
Example 8
| Source File: TestDictionaryLookup.java From dremio-oss with Apache License 2.0 | 4 votes |
|
@Test
public void testDictionaryLookup() throws Throwable {
try (final VectorContainer dict1 = new VectorContainer(getTestAllocator());
final VectorContainer dict2 = new VectorContainer(getTestAllocator());
final VectorContainer dict3 = new VectorContainer(getTestAllocator())) {
final Map<String, GlobalDictionaryFieldInfo> dictionaryFieldInfoMap = Maps.newHashMap();
final Field field1 = new Field(SchemaPath.getSimplePath("c0").getAsUnescapedPath(), true, new ArrowType.Int(64, true), null);
final BigIntVector longVector = dict1.addOrGet(field1);
longVector.allocateNew();
longVector.setSafe(0, 10L);
longVector.setSafe(1, 20L);
longVector.setSafe(2, 30L);
longVector.setSafe(3, 40L);
longVector.setSafe(4, 50L);
longVector.setValueCount(5);
dict1.setRecordCount(5);
dict1.buildSchema(BatchSchema.SelectionVectorMode.NONE);
dictionaryFieldInfoMap.put("c0", new GlobalDictionaryFieldInfo(0, "c0", null, field1.getType(), "local"));
final Field field2 = new Field(SchemaPath.getSimplePath("c1").getAsUnescapedPath(), true, new ArrowType.Binary(), null);
final VarBinaryVector binaryVector = dict2.addOrGet(field2);
binaryVector.allocateNew();
binaryVector.setSafe(0, "abc".getBytes(UTF8), 0, 3);
binaryVector.setSafe(1, "bcd".getBytes(UTF8), 0, 3);
binaryVector.setSafe(2, "cde".getBytes(UTF8), 0, 3);
binaryVector.setSafe(3, "def".getBytes(UTF8), 0, 3);
binaryVector.setSafe(4, "efg".getBytes(UTF8), 0, 3);
binaryVector.setValueCount(5);
dict2.setRecordCount(5);
dict2.buildSchema(BatchSchema.SelectionVectorMode.NONE);
dictionaryFieldInfoMap.put("c1", new GlobalDictionaryFieldInfo(0, "c1", null, field2.getType(), "local"));
final Field field3 = new Field(SchemaPath.getSimplePath("c2").getAsUnescapedPath(), true, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), null);
final Float8Vector doubleVector = dict3.addOrGet(field3);
doubleVector.allocateNew();
doubleVector.setSafe(0, 100.1);
doubleVector.setSafe(1, 200.2);
doubleVector.setSafe(2, 300.3);
doubleVector.setSafe(3, 400.4);
doubleVector.setSafe(4, 500.5);
doubleVector.setValueCount(5);
dict3.setRecordCount(5);
dict3.buildSchema(BatchSchema.SelectionVectorMode.NONE);
dictionaryFieldInfoMap.put("c2", new GlobalDictionaryFieldInfo(0, "c2", null, field3.getType(), "local"));
OperatorCreatorRegistry registry = Mockito.mock(OperatorCreatorRegistry.class);
Mockito.when(registry.getSingleInputOperator(Matchers.any(OperatorContext.class), Matchers.any(PhysicalOperator.class)))
.thenAnswer(new Answer<SingleInputOperator>() {
public SingleInputOperator answer(InvocationOnMock invocation) throws Exception {
Object[] args = invocation.getArguments();
DictionaryLookupOperator dictionaryLookupOperator = Mockito.spy(new DictionaryLookupOperator(
(OperatorContext)args[0], (DictionaryLookupPOP)args[1]));
Mockito.doReturn(dict1).when(dictionaryLookupOperator).loadDictionary(Matchers.eq("c0"));
Mockito.doReturn(dict2).when(dictionaryLookupOperator).loadDictionary(Matchers.eq("c1"));
Mockito.doReturn(dict3).when(dictionaryLookupOperator).loadDictionary(Matchers.eq("c2"));
return dictionaryLookupOperator;
}
});
BaseTestOperator.testContext.setRegistry(registry);
DictionaryLookupPOP lookup = new DictionaryLookupPOP(null, PROPS, null, dictionaryFieldInfoMap);
Table input = t(
th("c0", "c1", "c2"),
tr(0, 1, 2),
tr(1, 2, 0),
tr(2, 0, 1)
);
Table output = t(
th("c0", "c1", "c2"),
tr(10L, "bcd".getBytes(UTF8), 300.3),
tr(20L, "cde".getBytes(UTF8), 100.1),
tr(30L, "abc".getBytes(UTF8), 200.2)
);
validateSingle(lookup, DictionaryLookupOperator.class, input, output);
}
}
Example 9
| Source File: APIFieldDescriber.java From dremio-oss with Apache License 2.0 | 4 votes |
|
@Override
public Void visit(ArrowType.Binary binary) {
return writeString("VARBINARY");
}
