Java Code Examples for org.apache.flink.api.common.typeinfo.BasicTypeInfo#BOOLEAN_TYPE_INFO

@Test
public void testEitherHierarchy() {
	MapFunction<?, ?> function = new EitherMapper<Boolean>();
	TypeInformation<?> ti = TypeExtractor.getMapReturnTypes((MapFunction) function, BasicTypeInfo.BOOLEAN_TYPE_INFO);
	TypeInformation<?> expected = new EitherTypeInfo(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.BOOLEAN_TYPE_INFO);
	Assert.assertEquals(expected, ti);

	function = new EitherMapper2();
	ti = TypeExtractor.getMapReturnTypes((MapFunction) function, BasicTypeInfo.STRING_TYPE_INFO);
	expected = new EitherTypeInfo(BasicTypeInfo.STRING_TYPE_INFO, new TupleTypeInfo(BasicTypeInfo.INT_TYPE_INFO));
	Assert.assertEquals(expected, ti);

	function = new EitherMapper3();
	ti = TypeExtractor.getMapReturnTypes((MapFunction) function, expected);
	Assert.assertEquals(expected, ti);

	Either<String, Tuple1<Integer>> either = new Either2();
	ti = TypeExtractor.getForObject(either);
	Assert.assertEquals(expected, ti);
}
 

@Test
public void testEitherHierarchy() {
	MapFunction<?, ?> function = new EitherMapper<Boolean>();
	TypeInformation<?> ti = TypeExtractor.getMapReturnTypes((MapFunction) function, BasicTypeInfo.BOOLEAN_TYPE_INFO);
	TypeInformation<?> expected = new EitherTypeInfo(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.BOOLEAN_TYPE_INFO);
	Assert.assertEquals(expected, ti);

	function = new EitherMapper2();
	ti = TypeExtractor.getMapReturnTypes((MapFunction) function, BasicTypeInfo.STRING_TYPE_INFO);
	expected = new EitherTypeInfo(BasicTypeInfo.STRING_TYPE_INFO, new TupleTypeInfo(BasicTypeInfo.INT_TYPE_INFO));
	Assert.assertEquals(expected, ti);

	function = new EitherMapper3();
	ti = TypeExtractor.getMapReturnTypes((MapFunction) function, expected);
	Assert.assertEquals(expected, ti);

	Either<String, Tuple1<Integer>> either = new Either2();
	ti = TypeExtractor.getForObject(either);
	Assert.assertEquals(expected, ti);
}
 

@Test
public void testEither() {
	MapFunction<?, ?> function = new MapFunction<Either<String, Boolean>, Either<String, Boolean>>() {
		@Override
		public Either<String, Boolean> map(Either<String, Boolean> value) throws Exception {
			return null;
		}
	};
	TypeInformation<?> expected = new EitherTypeInfo(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.BOOLEAN_TYPE_INFO);
	TypeInformation<?> ti = TypeExtractor.getMapReturnTypes((MapFunction) function, expected);
	Assert.assertEquals(expected, ti);
}
 

@Nullable
private Tuple2<Column, Comparable> extractColumnAndLiteral(BinaryComparison comp) {
	TypeInformation<?> typeInfo = getLiteralType(comp);
	String columnName = getColumnName(comp);

	// fetch literal and ensure it is comparable
	Object value = getLiteral(comp);
	// validate that literal is comparable
	if (!(value instanceof Comparable)) {
		LOG.warn("Encountered a non-comparable literal of type {}." +
			"Cannot push predicate [{}] into ParquetTablesource." +
			"This is a bug and should be reported.", value.getClass().getCanonicalName(), comp);
		return null;
	}

	if (typeInfo == BasicTypeInfo.BYTE_TYPE_INFO ||
		typeInfo == BasicTypeInfo.SHORT_TYPE_INFO ||
		typeInfo == BasicTypeInfo.INT_TYPE_INFO) {
		return new Tuple2<>(FilterApi.intColumn(columnName), (Integer) value);
	} else if (typeInfo == BasicTypeInfo.LONG_TYPE_INFO) {
		return new Tuple2<>(FilterApi.longColumn(columnName), (Long) value);
	} else if (typeInfo == BasicTypeInfo.FLOAT_TYPE_INFO) {
		return new Tuple2<>(FilterApi.floatColumn(columnName), (Float) value);
	} else if (typeInfo == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
		return new Tuple2<>(FilterApi.booleanColumn(columnName), (Boolean) value);
	} else if (typeInfo == BasicTypeInfo.DOUBLE_TYPE_INFO) {
		return new Tuple2<>(FilterApi.doubleColumn(columnName), (Double) value);
	} else if (typeInfo == BasicTypeInfo.STRING_TYPE_INFO) {
		return new Tuple2<>(FilterApi.binaryColumn(columnName), Binary.fromString((String) value));
	} else {
		// unsupported type
		return null;
	}
}
 

@Test
public void testFunctionDependingOnInputWithTupleInput() {
	IdentityMapper2<Boolean> function = new IdentityMapper2<Boolean>();

	TypeInformation<Tuple2<Boolean, String>> inputType = new TupleTypeInfo<Tuple2<Boolean, String>>(BasicTypeInfo.BOOLEAN_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);

	TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, inputType);

	Assert.assertTrue(ti.isBasicType());
	Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, ti);
}
 

@Override
protected ListTypeInfo<?>[] getTestData() {
	return new ListTypeInfo<?>[] {
		new ListTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO),
		new ListTypeInfo<>(BasicTypeInfo.BOOLEAN_TYPE_INFO),
		new ListTypeInfo<>(Object.class),
	};
}
 

@Test
public void testEither() {
	MapFunction<?, ?> function = new MapFunction<Either<String, Boolean>, Either<String, Boolean>>() {
		@Override
		public Either<String, Boolean> map(Either<String, Boolean> value) throws Exception {
			return null;
		}
	};
	TypeInformation<?> expected = new EitherTypeInfo(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.BOOLEAN_TYPE_INFO);
	TypeInformation<?> ti = TypeExtractor.getMapReturnTypes((MapFunction) function, expected);
	Assert.assertEquals(expected, ti);
}
 

@Override
protected ListTypeInfo<?>[] getTestData() {
	return new ListTypeInfo<?>[] {
		new ListTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO),
		new ListTypeInfo<>(BasicTypeInfo.BOOLEAN_TYPE_INFO),
		new ListTypeInfo<>(Object.class),
	};
}
 

private TypeInformation getFieldType(HCatFieldSchema fieldSchema) {

		switch(fieldSchema.getType()) {
			case INT:
				return BasicTypeInfo.INT_TYPE_INFO;
			case TINYINT:
				return BasicTypeInfo.BYTE_TYPE_INFO;
			case SMALLINT:
				return BasicTypeInfo.SHORT_TYPE_INFO;
			case BIGINT:
				return BasicTypeInfo.LONG_TYPE_INFO;
			case BOOLEAN:
				return BasicTypeInfo.BOOLEAN_TYPE_INFO;
			case FLOAT:
				return BasicTypeInfo.FLOAT_TYPE_INFO;
			case DOUBLE:
				return BasicTypeInfo.DOUBLE_TYPE_INFO;
			case STRING:
				return BasicTypeInfo.STRING_TYPE_INFO;
			case BINARY:
				return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
			case ARRAY:
				return new GenericTypeInfo(List.class);
			case MAP:
				return new GenericTypeInfo(Map.class);
			case STRUCT:
				return new GenericTypeInfo(List.class);
			default:
				throw new IllegalArgumentException("Unknown data type \"" + fieldSchema.getType() + "\" encountered.");
		}
	}
 

@Override
protected RowTypeInfo[] getTestData() {
	return new RowTypeInfo[] {
		new RowTypeInfo(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO),
		new RowTypeInfo(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.BOOLEAN_TYPE_INFO),
		new RowTypeInfo(typeList),
		new RowTypeInfo(
			new TypeInformation[]{BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO},
			new String[]{"int", "int2"})
	};
}
 

private TypeInformation getFieldType(HCatFieldSchema fieldSchema) {

		switch(fieldSchema.getType()) {
			case INT:
				return BasicTypeInfo.INT_TYPE_INFO;
			case TINYINT:
				return BasicTypeInfo.BYTE_TYPE_INFO;
			case SMALLINT:
				return BasicTypeInfo.SHORT_TYPE_INFO;
			case BIGINT:
				return BasicTypeInfo.LONG_TYPE_INFO;
			case BOOLEAN:
				return BasicTypeInfo.BOOLEAN_TYPE_INFO;
			case FLOAT:
				return BasicTypeInfo.FLOAT_TYPE_INFO;
			case DOUBLE:
				return BasicTypeInfo.DOUBLE_TYPE_INFO;
			case STRING:
				return BasicTypeInfo.STRING_TYPE_INFO;
			case BINARY:
				return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
			case ARRAY:
				return new GenericTypeInfo(List.class);
			case MAP:
				return new GenericTypeInfo(Map.class);
			case STRUCT:
				return new GenericTypeInfo(List.class);
			default:
				throw new IllegalArgumentException("Unknown data type \"" + fieldSchema.getType() + "\" encountered.");
		}
	}
 

@Test
public void testEither() {
	MapFunction<?, ?> function = new MapFunction<Either<String, Boolean>, Either<String, Boolean>>() {
		@Override
		public Either<String, Boolean> map(Either<String, Boolean> value) throws Exception {
			return null;
		}
	};
	TypeInformation<?> expected = new EitherTypeInfo(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.BOOLEAN_TYPE_INFO);
	TypeInformation<?> ti = TypeExtractor.getMapReturnTypes((MapFunction) function, expected);
	Assert.assertEquals(expected, ti);
}
 

@Nullable
private Tuple2<Column, Comparable> extractColumnAndLiteral(BinaryComparison comp) {
	String columnName = getColumnName(comp);
	ColumnPath columnPath = ColumnPath.fromDotString(columnName);
	TypeInformation<?> typeInfo = null;
	try {
		Type type = parquetSchema.getType(columnPath.toArray());
		typeInfo = ParquetSchemaConverter.convertParquetTypeToTypeInfo(type);
	} catch (InvalidRecordException e) {
		LOG.error("Pushed predicate on undefined field name {} in schema", columnName);
		return null;
	}

	// fetch literal and ensure it is comparable
	Object value = getLiteral(comp);
	// validate that literal is comparable
	if (!(value instanceof Comparable)) {
		LOG.warn("Encountered a non-comparable literal of type {}." +
			"Cannot push predicate [{}] into ParquetTablesource." +
			"This is a bug and should be reported.", value.getClass().getCanonicalName(), comp);
		return null;
	}

	if (typeInfo == BasicTypeInfo.BYTE_TYPE_INFO ||
		typeInfo == BasicTypeInfo.SHORT_TYPE_INFO ||
		typeInfo == BasicTypeInfo.INT_TYPE_INFO) {
		return new Tuple2<>(FilterApi.intColumn(columnName), ((Number) value).intValue());
	} else if (typeInfo == BasicTypeInfo.LONG_TYPE_INFO) {
		return new Tuple2<>(FilterApi.longColumn(columnName), ((Number) value).longValue());
	} else if (typeInfo == BasicTypeInfo.FLOAT_TYPE_INFO) {
		return new Tuple2<>(FilterApi.floatColumn(columnName), ((Number) value).floatValue());
	} else if (typeInfo == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
		return new Tuple2<>(FilterApi.booleanColumn(columnName), (Boolean) value);
	} else if (typeInfo == BasicTypeInfo.DOUBLE_TYPE_INFO) {
		return new Tuple2<>(FilterApi.doubleColumn(columnName), ((Number) value).doubleValue());
	} else if (typeInfo == BasicTypeInfo.STRING_TYPE_INFO) {
		return new Tuple2<>(FilterApi.binaryColumn(columnName), Binary.fromString((String) value));
	} else {
		// unsupported type
		return null;
	}
}
 

@Override
protected TypeInformation<Boolean> getValueTypeInfo() {
	return BasicTypeInfo.BOOLEAN_TYPE_INFO;
}
 

/**
 * Converts an ORC schema to a Flink TypeInformation.
 *
 * @param schema The ORC schema.
 * @return The TypeInformation that corresponds to the ORC schema.
 */
static TypeInformation schemaToTypeInfo(TypeDescription schema) {
	switch (schema.getCategory()) {
		case BOOLEAN:
			return BasicTypeInfo.BOOLEAN_TYPE_INFO;
		case BYTE:
			return BasicTypeInfo.BYTE_TYPE_INFO;
		case SHORT:
			return BasicTypeInfo.SHORT_TYPE_INFO;
		case INT:
			return BasicTypeInfo.INT_TYPE_INFO;
		case LONG:
			return BasicTypeInfo.LONG_TYPE_INFO;
		case FLOAT:
			return BasicTypeInfo.FLOAT_TYPE_INFO;
		case DOUBLE:
			return BasicTypeInfo.DOUBLE_TYPE_INFO;
		case DECIMAL:
			return BasicTypeInfo.BIG_DEC_TYPE_INFO;
		case STRING:
		case CHAR:
		case VARCHAR:
			return BasicTypeInfo.STRING_TYPE_INFO;
		case DATE:
			return SqlTimeTypeInfo.DATE;
		case TIMESTAMP:
			return SqlTimeTypeInfo.TIMESTAMP;
		case BINARY:
			return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
		case STRUCT:
			List<TypeDescription> fieldSchemas = schema.getChildren();
			TypeInformation[] fieldTypes = new TypeInformation[fieldSchemas.size()];
			for (int i = 0; i < fieldSchemas.size(); i++) {
				fieldTypes[i] = schemaToTypeInfo(fieldSchemas.get(i));
			}
			String[] fieldNames = schema.getFieldNames().toArray(new String[]{});
			return new RowTypeInfo(fieldTypes, fieldNames);
		case LIST:
			TypeDescription elementSchema = schema.getChildren().get(0);
			TypeInformation<?> elementType = schemaToTypeInfo(elementSchema);
			// arrays of primitive types are handled as object arrays to support null values
			return ObjectArrayTypeInfo.getInfoFor(elementType);
		case MAP:
			TypeDescription keySchema = schema.getChildren().get(0);
			TypeDescription valSchema = schema.getChildren().get(1);
			TypeInformation<?> keyType = schemaToTypeInfo(keySchema);
			TypeInformation<?> valType = schemaToTypeInfo(valSchema);
			return new MapTypeInfo<>(keyType, valType);
		case UNION:
			throw new UnsupportedOperationException("UNION type is not supported yet.");
		default:
			throw new IllegalArgumentException("Unknown type " + schema);
	}
}
 

@Override
public TypeInformation<?> getResultType(Class<?>[] signature) {
	return BasicTypeInfo.BOOLEAN_TYPE_INFO;
}
 

@Test
public void testEmptyFields() throws Exception {
	String fileContent =
			",,,,,,,,\n" +
					",,,,,,,\n" +
					",,,,,,,,\n" +
					",,,,,,,\n" +
					",,,,,,,,\n" +
					",,,,,,,,\n" +
					",,,,,,,\n" +
					",,,,,,,,\n";

	FileInputSplit split = createTempFile(fileContent);

	TypeInformation[] fieldTypes = new TypeInformation[]{
			BasicTypeInfo.BOOLEAN_TYPE_INFO,
			BasicTypeInfo.BYTE_TYPE_INFO,
			BasicTypeInfo.DOUBLE_TYPE_INFO,
			BasicTypeInfo.FLOAT_TYPE_INFO,
			BasicTypeInfo.INT_TYPE_INFO,
			BasicTypeInfo.LONG_TYPE_INFO,
			BasicTypeInfo.SHORT_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO};

	RowCsvInputFormat.Builder builder = RowCsvInputFormat.builder(new RowTypeInfo(fieldTypes), PATH)
			.setFieldDelimiter(',')
			.setNullLiteral("");

	RowCsvInputFormat format = builder.build();
	format.configure(new Configuration());
	format.open(split);

	Row result = new Row(8);
	int linesCnt = fileContent.split("\n").length;

	for (int i = 0; i < linesCnt; i++) {
		result = format.nextRecord(result);
		assertNull(result.getField(i));
	}

	// ensure no more rows
	assertNull(format.nextRecord(result));
	assertTrue(format.reachedEnd());
}
 

@Test
public void testEmptyFields() throws Exception {
	String fileContent =
		",,,,,,,,\n" +
			",,,,,,,\n" +
			",,,,,,,,\n" +
			",,,,,,,\n" +
			",,,,,,,,\n" +
			",,,,,,,,\n" +
			",,,,,,,\n" +
			",,,,,,,,\n";

	FileInputSplit split = createTempFile(fileContent);

	TypeInformation[] fieldTypes = new TypeInformation[]{
		BasicTypeInfo.BOOLEAN_TYPE_INFO,
		BasicTypeInfo.BYTE_TYPE_INFO,
		BasicTypeInfo.DOUBLE_TYPE_INFO,
		BasicTypeInfo.FLOAT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.LONG_TYPE_INFO,
		BasicTypeInfo.SHORT_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO};

	RowCsvInputFormat format = new RowCsvInputFormat(PATH, fieldTypes, true);
	format.setFieldDelimiter(",");
	format.configure(new Configuration());
	format.open(split);

	Row result = new Row(8);
	int linesCnt = fileContent.split("\n").length;

	for (int i = 0; i < linesCnt; i++) {
		result = format.nextRecord(result);
		assertNull(result.getField(i));
	}

	// ensure no more rows
	assertNull(format.nextRecord(result));
	assertTrue(format.reachedEnd());
}
 

@Override
protected TypeInformation<Boolean> getValueTypeInfo() {
	return BasicTypeInfo.BOOLEAN_TYPE_INFO;
}
 

/**
 * Converts an ORC schema to a Flink TypeInformation.
 *
 * @param schema The ORC schema.
 * @return The TypeInformation that corresponds to the ORC schema.
 */
static TypeInformation schemaToTypeInfo(TypeDescription schema) {
	switch (schema.getCategory()) {
		case BOOLEAN:
			return BasicTypeInfo.BOOLEAN_TYPE_INFO;
		case BYTE:
			return BasicTypeInfo.BYTE_TYPE_INFO;
		case SHORT:
			return BasicTypeInfo.SHORT_TYPE_INFO;
		case INT:
			return BasicTypeInfo.INT_TYPE_INFO;
		case LONG:
			return BasicTypeInfo.LONG_TYPE_INFO;
		case FLOAT:
			return BasicTypeInfo.FLOAT_TYPE_INFO;
		case DOUBLE:
			return BasicTypeInfo.DOUBLE_TYPE_INFO;
		case DECIMAL:
			return BasicTypeInfo.BIG_DEC_TYPE_INFO;
		case STRING:
		case CHAR:
		case VARCHAR:
			return BasicTypeInfo.STRING_TYPE_INFO;
		case DATE:
			return SqlTimeTypeInfo.DATE;
		case TIMESTAMP:
			return SqlTimeTypeInfo.TIMESTAMP;
		case BINARY:
			return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
		case STRUCT:
			List<TypeDescription> fieldSchemas = schema.getChildren();
			TypeInformation[] fieldTypes = new TypeInformation[fieldSchemas.size()];
			for (int i = 0; i < fieldSchemas.size(); i++) {
				fieldTypes[i] = schemaToTypeInfo(fieldSchemas.get(i));
			}
			String[] fieldNames = schema.getFieldNames().toArray(new String[]{});
			return new RowTypeInfo(fieldTypes, fieldNames);
		case LIST:
			TypeDescription elementSchema = schema.getChildren().get(0);
			TypeInformation<?> elementType = schemaToTypeInfo(elementSchema);
			// arrays of primitive types are handled as object arrays to support null values
			return ObjectArrayTypeInfo.getInfoFor(elementType);
		case MAP:
			TypeDescription keySchema = schema.getChildren().get(0);
			TypeDescription valSchema = schema.getChildren().get(1);
			TypeInformation<?> keyType = schemaToTypeInfo(keySchema);
			TypeInformation<?> valType = schemaToTypeInfo(valSchema);
			return new MapTypeInfo<>(keyType, valType);
		case UNION:
			throw new UnsupportedOperationException("UNION type is not supported yet.");
		default:
			throw new IllegalArgumentException("Unknown type " + schema);
	}
}