Java Code Examples for org.apache.flink.api.common.typeinfo.BasicTypeInfo#SHORT_TYPE_INFO
The following examples show how to use
org.apache.flink.api.common.typeinfo.BasicTypeInfo#SHORT_TYPE_INFO .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: OrcTableSource.java From flink with Apache License 2.0 | 6 votes |
|
private PredicateLeaf.Type toOrcType(TypeInformation<?> type) {
if (type == BasicTypeInfo.BYTE_TYPE_INFO ||
type == BasicTypeInfo.SHORT_TYPE_INFO ||
type == BasicTypeInfo.INT_TYPE_INFO ||
type == BasicTypeInfo.LONG_TYPE_INFO) {
return PredicateLeaf.Type.LONG;
} else if (type == BasicTypeInfo.FLOAT_TYPE_INFO ||
type == BasicTypeInfo.DOUBLE_TYPE_INFO) {
return PredicateLeaf.Type.FLOAT;
} else if (type == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
return PredicateLeaf.Type.BOOLEAN;
} else if (type == BasicTypeInfo.STRING_TYPE_INFO) {
return PredicateLeaf.Type.STRING;
} else if (type == SqlTimeTypeInfo.TIMESTAMP) {
return PredicateLeaf.Type.TIMESTAMP;
} else if (type == SqlTimeTypeInfo.DATE) {
return PredicateLeaf.Type.DATE;
} else if (type == BasicTypeInfo.BIG_DEC_TYPE_INFO) {
return PredicateLeaf.Type.DECIMAL;
} else {
// unsupported type
return null;
}
}
Example 2
| Source File: OrcTableSource.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
private PredicateLeaf.Type toOrcType(TypeInformation<?> type) {
if (type == BasicTypeInfo.BYTE_TYPE_INFO ||
type == BasicTypeInfo.SHORT_TYPE_INFO ||
type == BasicTypeInfo.INT_TYPE_INFO ||
type == BasicTypeInfo.LONG_TYPE_INFO) {
return PredicateLeaf.Type.LONG;
} else if (type == BasicTypeInfo.FLOAT_TYPE_INFO ||
type == BasicTypeInfo.DOUBLE_TYPE_INFO) {
return PredicateLeaf.Type.FLOAT;
} else if (type == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
return PredicateLeaf.Type.BOOLEAN;
} else if (type == BasicTypeInfo.STRING_TYPE_INFO) {
return PredicateLeaf.Type.STRING;
} else if (type == SqlTimeTypeInfo.TIMESTAMP) {
return PredicateLeaf.Type.TIMESTAMP;
} else if (type == SqlTimeTypeInfo.DATE) {
return PredicateLeaf.Type.DATE;
} else if (type == BasicTypeInfo.BIG_DEC_TYPE_INFO) {
return PredicateLeaf.Type.DECIMAL;
} else {
// unsupported type
return null;
}
}
Example 3
| Source File: OrcTableSource.java From flink with Apache License 2.0 | 6 votes |
|
private PredicateLeaf.Type toOrcType(TypeInformation<?> type) {
if (type == BasicTypeInfo.BYTE_TYPE_INFO ||
type == BasicTypeInfo.SHORT_TYPE_INFO ||
type == BasicTypeInfo.INT_TYPE_INFO ||
type == BasicTypeInfo.LONG_TYPE_INFO) {
return PredicateLeaf.Type.LONG;
} else if (type == BasicTypeInfo.FLOAT_TYPE_INFO ||
type == BasicTypeInfo.DOUBLE_TYPE_INFO) {
return PredicateLeaf.Type.FLOAT;
} else if (type == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
return PredicateLeaf.Type.BOOLEAN;
} else if (type == BasicTypeInfo.STRING_TYPE_INFO) {
return PredicateLeaf.Type.STRING;
} else if (type == SqlTimeTypeInfo.TIMESTAMP) {
return PredicateLeaf.Type.TIMESTAMP;
} else if (type == SqlTimeTypeInfo.DATE) {
return PredicateLeaf.Type.DATE;
} else if (type == BasicTypeInfo.BIG_DEC_TYPE_INFO) {
return PredicateLeaf.Type.DECIMAL;
} else {
// unsupported type
return null;
}
}
Example 4
| Source File: ParquetTableSource.java From flink with Apache License 2.0 | 5 votes |
|
@Nullable
private Tuple2<Column, Comparable> extractColumnAndLiteral(BinaryComparison comp) {
TypeInformation<?> typeInfo = getLiteralType(comp);
String columnName = getColumnName(comp);
// fetch literal and ensure it is comparable
Object value = getLiteral(comp);
// validate that literal is comparable
if (!(value instanceof Comparable)) {
LOG.warn("Encountered a non-comparable literal of type {}." +
"Cannot push predicate [{}] into ParquetTablesource." +
"This is a bug and should be reported.", value.getClass().getCanonicalName(), comp);
return null;
}
if (typeInfo == BasicTypeInfo.BYTE_TYPE_INFO ||
typeInfo == BasicTypeInfo.SHORT_TYPE_INFO ||
typeInfo == BasicTypeInfo.INT_TYPE_INFO) {
return new Tuple2<>(FilterApi.intColumn(columnName), (Integer) value);
} else if (typeInfo == BasicTypeInfo.LONG_TYPE_INFO) {
return new Tuple2<>(FilterApi.longColumn(columnName), (Long) value);
} else if (typeInfo == BasicTypeInfo.FLOAT_TYPE_INFO) {
return new Tuple2<>(FilterApi.floatColumn(columnName), (Float) value);
} else if (typeInfo == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
return new Tuple2<>(FilterApi.booleanColumn(columnName), (Boolean) value);
} else if (typeInfo == BasicTypeInfo.DOUBLE_TYPE_INFO) {
return new Tuple2<>(FilterApi.doubleColumn(columnName), (Double) value);
} else if (typeInfo == BasicTypeInfo.STRING_TYPE_INFO) {
return new Tuple2<>(FilterApi.binaryColumn(columnName), Binary.fromString((String) value));
} else {
// unsupported type
return null;
}
}
Example 5
| Source File: HCatInputFormatBase.java From flink with Apache License 2.0 | 5 votes |
|
private TypeInformation getFieldType(HCatFieldSchema fieldSchema) {
switch(fieldSchema.getType()) {
case INT:
return BasicTypeInfo.INT_TYPE_INFO;
case TINYINT:
return BasicTypeInfo.BYTE_TYPE_INFO;
case SMALLINT:
return BasicTypeInfo.SHORT_TYPE_INFO;
case BIGINT:
return BasicTypeInfo.LONG_TYPE_INFO;
case BOOLEAN:
return BasicTypeInfo.BOOLEAN_TYPE_INFO;
case FLOAT:
return BasicTypeInfo.FLOAT_TYPE_INFO;
case DOUBLE:
return BasicTypeInfo.DOUBLE_TYPE_INFO;
case STRING:
return BasicTypeInfo.STRING_TYPE_INFO;
case BINARY:
return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
case ARRAY:
return new GenericTypeInfo(List.class);
case MAP:
return new GenericTypeInfo(Map.class);
case STRUCT:
return new GenericTypeInfo(List.class);
default:
throw new IllegalArgumentException("Unknown data type \"" + fieldSchema.getType() + "\" encountered.");
}
}
Example 6
| Source File: LegacyRowSerializerTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testRowSerializerWithComplexTypes() {
RowTypeInfo typeInfo = new RowTypeInfo(
BasicTypeInfo.INT_TYPE_INFO,
BasicTypeInfo.DOUBLE_TYPE_INFO,
BasicTypeInfo.STRING_TYPE_INFO,
new TupleTypeInfo<Tuple3<Integer, Boolean, Short>>(
BasicTypeInfo.INT_TYPE_INFO,
BasicTypeInfo.BOOLEAN_TYPE_INFO,
BasicTypeInfo.SHORT_TYPE_INFO),
TypeExtractor.createTypeInfo(MyPojo.class));
MyPojo testPojo1 = new MyPojo();
testPojo1.name = null;
MyPojo testPojo2 = new MyPojo();
testPojo2.name = "Test1";
MyPojo testPojo3 = new MyPojo();
testPojo3.name = "Test2";
Row[] data = new Row[]{
createRow(null, null, null, null, null),
createRow(0, null, null, null, null),
createRow(0, 0.0, null, null, null),
createRow(0, 0.0, "a", null, null),
createRow(1, 0.0, "a", null, null),
createRow(1, 1.0, "a", null, null),
createRow(1, 1.0, "b", null, null),
createRow(1, 1.0, "b", new Tuple3<>(1, false, (short) 2), null),
createRow(1, 1.0, "b", new Tuple3<>(2, false, (short) 2), null),
createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 2), null),
createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), null),
createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo1),
createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo2),
createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo3)
};
TypeSerializer<Row> serializer = typeInfo.createLegacySerializer(new ExecutionConfig());
RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, data);
testInstance.testAll();
}
Example 7
| Source File: HCatInputFormatBase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private TypeInformation getFieldType(HCatFieldSchema fieldSchema) {
switch(fieldSchema.getType()) {
case INT:
return BasicTypeInfo.INT_TYPE_INFO;
case TINYINT:
return BasicTypeInfo.BYTE_TYPE_INFO;
case SMALLINT:
return BasicTypeInfo.SHORT_TYPE_INFO;
case BIGINT:
return BasicTypeInfo.LONG_TYPE_INFO;
case BOOLEAN:
return BasicTypeInfo.BOOLEAN_TYPE_INFO;
case FLOAT:
return BasicTypeInfo.FLOAT_TYPE_INFO;
case DOUBLE:
return BasicTypeInfo.DOUBLE_TYPE_INFO;
case STRING:
return BasicTypeInfo.STRING_TYPE_INFO;
case BINARY:
return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
case ARRAY:
return new GenericTypeInfo(List.class);
case MAP:
return new GenericTypeInfo(Map.class);
case STRUCT:
return new GenericTypeInfo(List.class);
default:
throw new IllegalArgumentException("Unknown data type \"" + fieldSchema.getType() + "\" encountered.");
}
}
Example 8
| Source File: HCatInputFormatBase.java From flink with Apache License 2.0 | 5 votes |
|
private TypeInformation getFieldType(HCatFieldSchema fieldSchema) {
switch(fieldSchema.getType()) {
case INT:
return BasicTypeInfo.INT_TYPE_INFO;
case TINYINT:
return BasicTypeInfo.BYTE_TYPE_INFO;
case SMALLINT:
return BasicTypeInfo.SHORT_TYPE_INFO;
case BIGINT:
return BasicTypeInfo.LONG_TYPE_INFO;
case BOOLEAN:
return BasicTypeInfo.BOOLEAN_TYPE_INFO;
case FLOAT:
return BasicTypeInfo.FLOAT_TYPE_INFO;
case DOUBLE:
return BasicTypeInfo.DOUBLE_TYPE_INFO;
case STRING:
return BasicTypeInfo.STRING_TYPE_INFO;
case BINARY:
return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
case ARRAY:
return new GenericTypeInfo(List.class);
case MAP:
return new GenericTypeInfo(Map.class);
case STRUCT:
return new GenericTypeInfo(List.class);
default:
throw new IllegalArgumentException("Unknown data type \"" + fieldSchema.getType() + "\" encountered.");
}
}
Example 9
| Source File: RowSerializerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testRowSerializerWithComplexTypes() {
TypeInformation<Row> typeInfo = new RowTypeInfo(
BasicTypeInfo.INT_TYPE_INFO,
BasicTypeInfo.DOUBLE_TYPE_INFO,
BasicTypeInfo.STRING_TYPE_INFO,
new TupleTypeInfo<Tuple3<Integer, Boolean, Short>>(
BasicTypeInfo.INT_TYPE_INFO,
BasicTypeInfo.BOOLEAN_TYPE_INFO,
BasicTypeInfo.SHORT_TYPE_INFO),
TypeExtractor.createTypeInfo(MyPojo.class));
MyPojo testPojo1 = new MyPojo();
testPojo1.name = null;
MyPojo testPojo2 = new MyPojo();
testPojo2.name = "Test1";
MyPojo testPojo3 = new MyPojo();
testPojo3.name = "Test2";
Row[] data = new Row[]{
createRow(null, null, null, null, null),
createRow(0, null, null, null, null),
createRow(0, 0.0, null, null, null),
createRow(0, 0.0, "a", null, null),
createRow(1, 0.0, "a", null, null),
createRow(1, 1.0, "a", null, null),
createRow(1, 1.0, "b", null, null),
createRow(1, 1.0, "b", new Tuple3<>(1, false, (short) 2), null),
createRow(1, 1.0, "b", new Tuple3<>(2, false, (short) 2), null),
createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 2), null),
createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), null),
createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo1),
createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo2),
createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo3)
};
TypeSerializer<Row> serializer = typeInfo.createSerializer(new ExecutionConfig());
RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, data);
testInstance.testAll();
}
Example 10
| Source File: OrcBatchReader.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Converts an ORC schema to a Flink TypeInformation.
*
* @param schema The ORC schema.
* @return The TypeInformation that corresponds to the ORC schema.
*/
static TypeInformation schemaToTypeInfo(TypeDescription schema) {
switch (schema.getCategory()) {
case BOOLEAN:
return BasicTypeInfo.BOOLEAN_TYPE_INFO;
case BYTE:
return BasicTypeInfo.BYTE_TYPE_INFO;
case SHORT:
return BasicTypeInfo.SHORT_TYPE_INFO;
case INT:
return BasicTypeInfo.INT_TYPE_INFO;
case LONG:
return BasicTypeInfo.LONG_TYPE_INFO;
case FLOAT:
return BasicTypeInfo.FLOAT_TYPE_INFO;
case DOUBLE:
return BasicTypeInfo.DOUBLE_TYPE_INFO;
case DECIMAL:
return BasicTypeInfo.BIG_DEC_TYPE_INFO;
case STRING:
case CHAR:
case VARCHAR:
return BasicTypeInfo.STRING_TYPE_INFO;
case DATE:
return SqlTimeTypeInfo.DATE;
case TIMESTAMP:
return SqlTimeTypeInfo.TIMESTAMP;
case BINARY:
return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
case STRUCT:
List<TypeDescription> fieldSchemas = schema.getChildren();
TypeInformation[] fieldTypes = new TypeInformation[fieldSchemas.size()];
for (int i = 0; i < fieldSchemas.size(); i++) {
fieldTypes[i] = schemaToTypeInfo(fieldSchemas.get(i));
}
String[] fieldNames = schema.getFieldNames().toArray(new String[]{});
return new RowTypeInfo(fieldTypes, fieldNames);
case LIST:
TypeDescription elementSchema = schema.getChildren().get(0);
TypeInformation<?> elementType = schemaToTypeInfo(elementSchema);
// arrays of primitive types are handled as object arrays to support null values
return ObjectArrayTypeInfo.getInfoFor(elementType);
case MAP:
TypeDescription keySchema = schema.getChildren().get(0);
TypeDescription valSchema = schema.getChildren().get(1);
TypeInformation<?> keyType = schemaToTypeInfo(keySchema);
TypeInformation<?> valType = schemaToTypeInfo(valSchema);
return new MapTypeInfo<>(keyType, valType);
case UNION:
throw new UnsupportedOperationException("UNION type is not supported yet.");
default:
throw new IllegalArgumentException("Unknown type " + schema);
}
}
Example 11
| Source File: RowCsvInputFormatTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testEmptyFields() throws Exception {
String fileContent =
",,,,,,,,\n" +
",,,,,,,\n" +
",,,,,,,,\n" +
",,,,,,,\n" +
",,,,,,,,\n" +
",,,,,,,,\n" +
",,,,,,,\n" +
",,,,,,,,\n";
FileInputSplit split = createTempFile(fileContent);
TypeInformation[] fieldTypes = new TypeInformation[]{
BasicTypeInfo.BOOLEAN_TYPE_INFO,
BasicTypeInfo.BYTE_TYPE_INFO,
BasicTypeInfo.DOUBLE_TYPE_INFO,
BasicTypeInfo.FLOAT_TYPE_INFO,
BasicTypeInfo.INT_TYPE_INFO,
BasicTypeInfo.LONG_TYPE_INFO,
BasicTypeInfo.SHORT_TYPE_INFO,
BasicTypeInfo.STRING_TYPE_INFO};
RowCsvInputFormat format = new RowCsvInputFormat(PATH, fieldTypes, true);
format.setFieldDelimiter(",");
format.configure(new Configuration());
format.open(split);
Row result = new Row(8);
int linesCnt = fileContent.split("\n").length;
for (int i = 0; i < linesCnt; i++) {
result = format.nextRecord(result);
assertNull(result.getField(i));
}
// ensure no more rows
assertNull(format.nextRecord(result));
assertTrue(format.reachedEnd());
}
Example 12
| Source File: MaxWithRetractAggFunction.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected TypeInformation<Short> getValueTypeInfo() {
return BasicTypeInfo.SHORT_TYPE_INFO;
}
Example 13
| Source File: RowCsvInputFormatTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testEmptyFields() throws Exception {
String fileContent =
",,,,,,,,\n" +
",,,,,,,\n" +
",,,,,,,,\n" +
",,,,,,,\n" +
",,,,,,,,\n" +
",,,,,,,,\n" +
",,,,,,,\n" +
",,,,,,,,\n";
FileInputSplit split = createTempFile(fileContent);
TypeInformation[] fieldTypes = new TypeInformation[]{
BasicTypeInfo.BOOLEAN_TYPE_INFO,
BasicTypeInfo.BYTE_TYPE_INFO,
BasicTypeInfo.DOUBLE_TYPE_INFO,
BasicTypeInfo.FLOAT_TYPE_INFO,
BasicTypeInfo.INT_TYPE_INFO,
BasicTypeInfo.LONG_TYPE_INFO,
BasicTypeInfo.SHORT_TYPE_INFO,
BasicTypeInfo.STRING_TYPE_INFO};
RowCsvInputFormat format = new RowCsvInputFormat(PATH, fieldTypes, true);
format.setFieldDelimiter(",");
format.configure(new Configuration());
format.open(split);
Row result = new Row(8);
int linesCnt = fileContent.split("\n").length;
for (int i = 0; i < linesCnt; i++) {
result = format.nextRecord(result);
assertNull(result.getField(i));
}
// ensure no more rows
assertNull(format.nextRecord(result));
assertTrue(format.reachedEnd());
}
Example 14
| Source File: RowCsvInputFormatTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testEmptyFields() throws Exception {
String fileContent =
",,,,,,,,\n" +
",,,,,,,\n" +
",,,,,,,,\n" +
",,,,,,,\n" +
",,,,,,,,\n" +
",,,,,,,,\n" +
",,,,,,,\n" +
",,,,,,,,\n";
FileInputSplit split = createTempFile(fileContent);
TypeInformation[] fieldTypes = new TypeInformation[]{
BasicTypeInfo.BOOLEAN_TYPE_INFO,
BasicTypeInfo.BYTE_TYPE_INFO,
BasicTypeInfo.DOUBLE_TYPE_INFO,
BasicTypeInfo.FLOAT_TYPE_INFO,
BasicTypeInfo.INT_TYPE_INFO,
BasicTypeInfo.LONG_TYPE_INFO,
BasicTypeInfo.SHORT_TYPE_INFO,
BasicTypeInfo.STRING_TYPE_INFO};
RowCsvInputFormat format = new RowCsvInputFormat(PATH, fieldTypes, true);
format.setFieldDelimiter(",");
format.configure(new Configuration());
format.open(split);
Row result = new Row(8);
int linesCnt = fileContent.split("\n").length;
for (int i = 0; i < linesCnt; i++) {
result = format.nextRecord(result);
assertNull(result.getField(i));
}
// ensure no more rows
assertNull(format.nextRecord(result));
assertTrue(format.reachedEnd());
}
Example 15
| Source File: MinWithRetractAggFunction.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected TypeInformation<Short> getValueTypeInfo() {
return BasicTypeInfo.SHORT_TYPE_INFO;
}
Example 16
| Source File: MaxWithRetractAggFunction.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected TypeInformation<Short> getValueTypeInfo() {
return BasicTypeInfo.SHORT_TYPE_INFO;
}
Example 17
| Source File: RowCsvInputFormatTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testEmptyFields() throws Exception {
String fileContent =
",,,,,,,,\n" +
",,,,,,,\n" +
",,,,,,,,\n" +
",,,,,,,\n" +
",,,,,,,,\n" +
",,,,,,,,\n" +
",,,,,,,\n" +
",,,,,,,,\n";
FileInputSplit split = createTempFile(fileContent);
TypeInformation[] fieldTypes = new TypeInformation[]{
BasicTypeInfo.BOOLEAN_TYPE_INFO,
BasicTypeInfo.BYTE_TYPE_INFO,
BasicTypeInfo.DOUBLE_TYPE_INFO,
BasicTypeInfo.FLOAT_TYPE_INFO,
BasicTypeInfo.INT_TYPE_INFO,
BasicTypeInfo.LONG_TYPE_INFO,
BasicTypeInfo.SHORT_TYPE_INFO,
BasicTypeInfo.STRING_TYPE_INFO};
RowCsvInputFormat.Builder builder = RowCsvInputFormat.builder(new RowTypeInfo(fieldTypes), PATH)
.setFieldDelimiter(',')
.setNullLiteral("");
RowCsvInputFormat format = builder.build();
format.configure(new Configuration());
format.open(split);
Row result = new Row(8);
int linesCnt = fileContent.split("\n").length;
for (int i = 0; i < linesCnt; i++) {
result = format.nextRecord(result);
assertNull(result.getField(i));
}
// ensure no more rows
assertNull(format.nextRecord(result));
assertTrue(format.reachedEnd());
}
Example 18
| Source File: ParquetTableSource.java From flink with Apache License 2.0 | 4 votes |
|
@Nullable
private Tuple2<Column, Comparable> extractColumnAndLiteral(BinaryComparison comp) {
String columnName = getColumnName(comp);
ColumnPath columnPath = ColumnPath.fromDotString(columnName);
TypeInformation<?> typeInfo = null;
try {
Type type = parquetSchema.getType(columnPath.toArray());
typeInfo = ParquetSchemaConverter.convertParquetTypeToTypeInfo(type);
} catch (InvalidRecordException e) {
LOG.error("Pushed predicate on undefined field name {} in schema", columnName);
return null;
}
// fetch literal and ensure it is comparable
Object value = getLiteral(comp);
// validate that literal is comparable
if (!(value instanceof Comparable)) {
LOG.warn("Encountered a non-comparable literal of type {}." +
"Cannot push predicate [{}] into ParquetTablesource." +
"This is a bug and should be reported.", value.getClass().getCanonicalName(), comp);
return null;
}
if (typeInfo == BasicTypeInfo.BYTE_TYPE_INFO ||
typeInfo == BasicTypeInfo.SHORT_TYPE_INFO ||
typeInfo == BasicTypeInfo.INT_TYPE_INFO) {
return new Tuple2<>(FilterApi.intColumn(columnName), ((Number) value).intValue());
} else if (typeInfo == BasicTypeInfo.LONG_TYPE_INFO) {
return new Tuple2<>(FilterApi.longColumn(columnName), ((Number) value).longValue());
} else if (typeInfo == BasicTypeInfo.FLOAT_TYPE_INFO) {
return new Tuple2<>(FilterApi.floatColumn(columnName), ((Number) value).floatValue());
} else if (typeInfo == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
return new Tuple2<>(FilterApi.booleanColumn(columnName), (Boolean) value);
} else if (typeInfo == BasicTypeInfo.DOUBLE_TYPE_INFO) {
return new Tuple2<>(FilterApi.doubleColumn(columnName), ((Number) value).doubleValue());
} else if (typeInfo == BasicTypeInfo.STRING_TYPE_INFO) {
return new Tuple2<>(FilterApi.binaryColumn(columnName), Binary.fromString((String) value));
} else {
// unsupported type
return null;
}
}
Example 19
| Source File: OrcBatchReader.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Converts an ORC schema to a Flink TypeInformation.
*
* @param schema The ORC schema.
* @return The TypeInformation that corresponds to the ORC schema.
*/
static TypeInformation schemaToTypeInfo(TypeDescription schema) {
switch (schema.getCategory()) {
case BOOLEAN:
return BasicTypeInfo.BOOLEAN_TYPE_INFO;
case BYTE:
return BasicTypeInfo.BYTE_TYPE_INFO;
case SHORT:
return BasicTypeInfo.SHORT_TYPE_INFO;
case INT:
return BasicTypeInfo.INT_TYPE_INFO;
case LONG:
return BasicTypeInfo.LONG_TYPE_INFO;
case FLOAT:
return BasicTypeInfo.FLOAT_TYPE_INFO;
case DOUBLE:
return BasicTypeInfo.DOUBLE_TYPE_INFO;
case DECIMAL:
return BasicTypeInfo.BIG_DEC_TYPE_INFO;
case STRING:
case CHAR:
case VARCHAR:
return BasicTypeInfo.STRING_TYPE_INFO;
case DATE:
return SqlTimeTypeInfo.DATE;
case TIMESTAMP:
return SqlTimeTypeInfo.TIMESTAMP;
case BINARY:
return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
case STRUCT:
List<TypeDescription> fieldSchemas = schema.getChildren();
TypeInformation[] fieldTypes = new TypeInformation[fieldSchemas.size()];
for (int i = 0; i < fieldSchemas.size(); i++) {
fieldTypes[i] = schemaToTypeInfo(fieldSchemas.get(i));
}
String[] fieldNames = schema.getFieldNames().toArray(new String[]{});
return new RowTypeInfo(fieldTypes, fieldNames);
case LIST:
TypeDescription elementSchema = schema.getChildren().get(0);
TypeInformation<?> elementType = schemaToTypeInfo(elementSchema);
// arrays of primitive types are handled as object arrays to support null values
return ObjectArrayTypeInfo.getInfoFor(elementType);
case MAP:
TypeDescription keySchema = schema.getChildren().get(0);
TypeDescription valSchema = schema.getChildren().get(1);
TypeInformation<?> keyType = schemaToTypeInfo(keySchema);
TypeInformation<?> valType = schemaToTypeInfo(valSchema);
return new MapTypeInfo<>(keyType, valType);
case UNION:
throw new UnsupportedOperationException("UNION type is not supported yet.");
default:
throw new IllegalArgumentException("Unknown type " + schema);
}
}
Example 20
| Source File: OrcBatchReader.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
/**
* Converts an ORC schema to a Flink TypeInformation.
*
* @param schema The ORC schema.
* @return The TypeInformation that corresponds to the ORC schema.
*/
static TypeInformation schemaToTypeInfo(TypeDescription schema) {
switch (schema.getCategory()) {
case BOOLEAN:
return BasicTypeInfo.BOOLEAN_TYPE_INFO;
case BYTE:
return BasicTypeInfo.BYTE_TYPE_INFO;
case SHORT:
return BasicTypeInfo.SHORT_TYPE_INFO;
case INT:
return BasicTypeInfo.INT_TYPE_INFO;
case LONG:
return BasicTypeInfo.LONG_TYPE_INFO;
case FLOAT:
return BasicTypeInfo.FLOAT_TYPE_INFO;
case DOUBLE:
return BasicTypeInfo.DOUBLE_TYPE_INFO;
case DECIMAL:
return BasicTypeInfo.BIG_DEC_TYPE_INFO;
case STRING:
case CHAR:
case VARCHAR:
return BasicTypeInfo.STRING_TYPE_INFO;
case DATE:
return SqlTimeTypeInfo.DATE;
case TIMESTAMP:
return SqlTimeTypeInfo.TIMESTAMP;
case BINARY:
return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
case STRUCT:
List<TypeDescription> fieldSchemas = schema.getChildren();
TypeInformation[] fieldTypes = new TypeInformation[fieldSchemas.size()];
for (int i = 0; i < fieldSchemas.size(); i++) {
fieldTypes[i] = schemaToTypeInfo(fieldSchemas.get(i));
}
String[] fieldNames = schema.getFieldNames().toArray(new String[]{});
return new RowTypeInfo(fieldTypes, fieldNames);
case LIST:
TypeDescription elementSchema = schema.getChildren().get(0);
TypeInformation<?> elementType = schemaToTypeInfo(elementSchema);
// arrays of primitive types are handled as object arrays to support null values
return ObjectArrayTypeInfo.getInfoFor(elementType);
case MAP:
TypeDescription keySchema = schema.getChildren().get(0);
TypeDescription valSchema = schema.getChildren().get(1);
TypeInformation<?> keyType = schemaToTypeInfo(keySchema);
TypeInformation<?> valType = schemaToTypeInfo(valSchema);
return new MapTypeInfo<>(keyType, valType);
case UNION:
throw new UnsupportedOperationException("UNION type is not supported yet.");
default:
throw new IllegalArgumentException("Unknown type " + schema);
}
}
