Java Code Examples for org.apache.flink.api.java.typeutils.TypeExtractor#createTypeInfo()

/**
 * Create a new {@code StateDescriptor} with the given name and the given type information.
 *
 * <p>If this constructor fails (because it is not possible to describe the type via a class),
 * consider using the {@link #StateDescriptor(String, TypeInformation, Object)} constructor.
 *
 * @param name The name of the {@code StateDescriptor}.
 * @param type The class of the type of values in the state.
 * @param defaultValue The default value that will be set when requesting state without setting
 *                     a value before.
 */
protected StateDescriptor(String name, Class<T> type, @Nullable T defaultValue) {
	this.name = checkNotNull(name, "name must not be null");
	checkNotNull(type, "type class must not be null");

	try {
		this.typeInfo = TypeExtractor.createTypeInfo(type);
	} catch (Exception e) {
		throw new RuntimeException(
				"Could not create the type information for '" + type.getName() + "'. " +
				"The most common reason is failure to infer the generic type information, due to Java's type erasure. " +
				"In that case, please pass a 'TypeHint' instead of a class to describe the type. " +
				"For example, to describe 'Tuple2<String, String>' as a generic type, use " +
				"'new PravegaDeserializationSchema<>(new TypeHint<Tuple2<String, String>>(){}, serializer);'", e);
	}

	this.defaultValue = defaultValue;
}
 

@Test
public void testPojoTypeWithMappingInformation() throws Exception {
	File tempFile = File.createTempFile("CsvReaderPojoType", "tmp");
	tempFile.deleteOnExit();
	tempFile.setWritable(true);

	OutputStreamWriter wrt = new OutputStreamWriter(new FileOutputStream(tempFile));
	wrt.write("123,3.123,AAA,BBB\n");
	wrt.write("456,1.123,BBB,AAA\n");
	wrt.close();

	@SuppressWarnings("unchecked")
	PojoTypeInfo<PojoItem> typeInfo = (PojoTypeInfo<PojoItem>) TypeExtractor.createTypeInfo(PojoItem.class);
	CsvInputFormat<PojoItem> inputFormat = new PojoCsvInputFormat<PojoItem>(new Path(tempFile.toURI().toString()), typeInfo, new String[]{"field1", "field3", "field2", "field4"});

	inputFormat.configure(new Configuration());
	FileInputSplit[] splits = inputFormat.createInputSplits(1);

	inputFormat.open(splits[0]);

	validatePojoItem(inputFormat);
}
 

/**
 * Apply a function to the attribute of each vertex in the graph.
 *
 * @param mapper the map function to apply.
 * @return a new graph
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public <NV> Graph<K, NV, EV> mapVertices(final MapFunction<Vertex<K, VV>, NV> mapper) {

	TypeInformation<K> keyType = ((TupleTypeInfo<?>) vertices.getType()).getTypeAt(0);

	TypeInformation<NV> valueType;

	if (mapper instanceof ResultTypeQueryable) {
		valueType = ((ResultTypeQueryable) mapper).getProducedType();
	} else {
		valueType = TypeExtractor.createTypeInfo(MapFunction.class, mapper.getClass(), 1, vertices.getType(), null);
	}

	TypeInformation<Vertex<K, NV>> returnType = (TypeInformation<Vertex<K, NV>>) new TupleTypeInfo(
			Vertex.class, keyType, valueType);

	return mapVertices(mapper, returnType);
}
 

/**
 * Tries to infer the TypeInformation of an AggregateFunction's accumulator type.
 *
 * @param tableFunction The TableFunction for which the accumulator type is inferred.
 * @param scalaType The implicitly inferred type of the accumulator type.
 * @return The inferred accumulator type of the AggregateFunction.
 */
public static <T> TypeInformation<T> getReturnTypeOfTableFunction(
		TableFunction<T> tableFunction,
		TypeInformation<T> scalaType) {

	TypeInformation<T> userProvidedType = tableFunction.getResultType();
	if (userProvidedType != null) {
		return userProvidedType;
	} else if (scalaType != null) {
		return scalaType;
	} else {
		return TypeExtractor.createTypeInfo(
			tableFunction,
			TableFunction.class,
			tableFunction.getClass(),
			0);
	}
}
 

private static TypeInformation<?> getFlinkType(Type type)
{
    if (type instanceof ParameterizedType && ((ParameterizedType) type).getRawType() == Map.class) {
        Type[] arguments = ((ParameterizedType) type).getActualTypeArguments();
        Type valueType = arguments[1];
        TypeInformation<?> valueInfo = getFlinkType(valueType);
        return new MapTypeInfo<>(TypeExtractor.createTypeInfo(arguments[0]), valueInfo);
    }
    else if (type instanceof ParameterizedType && ((ParameterizedType) type).getRawType() == List.class) {
        TypeInformation<?> typeInformation = getFlinkType(((ParameterizedType) type).getActualTypeArguments()[0]);
        if (typeInformation.isBasicType() && typeInformation != Types.STRING) {
            return Types.PRIMITIVE_ARRAY(typeInformation);
        }
        else {
            return Types.OBJECT_ARRAY(typeInformation);
        }
    }
    else {
        return TypeExtractor.createTypeInfo(type);
    }
}
 

/**
 * This test validates proper serialization with specific (generated POJO) types.
 */
@Test
public void testDeserializeToSpecificType() throws IOException {

	DatumReader<User> datumReader = new SpecificDatumReader<>(userSchema);

	try (FileReader<User> dataFileReader = DataFileReader.openReader(testFile, datumReader)) {
		User rec = dataFileReader.next();

		// check if record has been read correctly
		assertNotNull(rec);
		assertEquals("name not equal", TEST_NAME, rec.get("name").toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), rec.get("type_enum").toString());

		// now serialize it with our framework:
		ExecutionConfig ec = new ExecutionConfig();
		TypeInformation<User> te = TypeExtractor.createTypeInfo(User.class);

		assertEquals(AvroTypeInfo.class, te.getClass());
		TypeSerializer<User> tser = te.createSerializer(ec);

		ByteArrayOutputStream out = new ByteArrayOutputStream();
		try (DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out)) {
			tser.serialize(rec, outView);
		}

		User newRec;
		try (DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(
				new ByteArrayInputStream(out.toByteArray()))) {
			newRec = tser.deserialize(inView);
		}

		// check if it is still the same
		assertNotNull(newRec);
		assertEquals("name not equal", TEST_NAME, newRec.getName().toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), newRec.getTypeEnum().toString());
	}
}
 

/**
 * Registers the given type with the serialization stack. If the type is eventually
 * serialized as a POJO, then the type is registered with the POJO serializer. If the
 * type ends up being serialized with Kryo, then it will be registered at Kryo to make
 * sure that only tags are written.
 *
 * @param type The class of the type to register.
 */
public void registerType(Class<?> type) {
	if (type == null) {
		throw new NullPointerException("Cannot register null type class.");
	}

	TypeInformation<?> typeInfo = TypeExtractor.createTypeInfo(type);

	if (typeInfo instanceof PojoTypeInfo) {
		config.registerPojoType(type);
	} else {
		config.registerKryoType(type);
	}
}
 

/**
 * Creates a graph from a DataSet of edges.
 * Vertices are created automatically and their values are set
 * by applying the provided map function to the vertex IDs.
 *
 * @param edges a DataSet of edges.
 * @param vertexValueInitializer the mapper function that initializes the vertex values.
 * It allows to apply a map transformation on the vertex ID to produce an initial vertex value.
 * @param context the flink execution environment.
 * @return the newly created graph.
 */
public static <K, VV, EV> Graph<K, VV, EV> fromDataSet(DataSet<Edge<K, EV>> edges,
		final MapFunction<K, VV> vertexValueInitializer, ExecutionEnvironment context) {

	TypeInformation<K> keyType = ((TupleTypeInfo<?>) edges.getType()).getTypeAt(0);

	TypeInformation<VV> valueType = TypeExtractor.createTypeInfo(
			MapFunction.class, vertexValueInitializer.getClass(), 1, keyType, null);

	@SuppressWarnings({ "unchecked", "rawtypes" })
	TypeInformation<Vertex<K, VV>> returnType = (TypeInformation<Vertex<K, VV>>) new TupleTypeInfo(
			Vertex.class, keyType, valueType);

	DataSet<Vertex<K, VV>> vertices = edges
		.flatMap(new EmitSrcAndTargetAsTuple1<>())
			.name("Source and target IDs")
		.distinct()
			.name("IDs")
		.map(new MapFunction<Tuple1<K>, Vertex<K, VV>>() {
			private Vertex<K, VV> output = new Vertex<>();

			public Vertex<K, VV> map(Tuple1<K> value) throws Exception {
				output.f0 = value.f0;
				output.f1 = vertexValueInitializer.map(value.f0);
				return output;
			}
		}).returns(returnType).withForwardedFields("f0").name("Initialize vertex values");

	return new Graph<>(vertices, edges, context);
}
 

/**
 * This test validates proper serialization with specific (generated POJO) types.
 */
@Test
public void testDeserializeToSpecificType() throws IOException {

	DatumReader<User> datumReader = new SpecificDatumReader<>(userSchema);

	try (FileReader<User> dataFileReader = DataFileReader.openReader(testFile, datumReader)) {
		User rec = dataFileReader.next();

		// check if record has been read correctly
		assertNotNull(rec);
		assertEquals("name not equal", TEST_NAME, rec.get("name").toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), rec.get("type_enum").toString());

		// now serialize it with our framework:
		ExecutionConfig ec = new ExecutionConfig();
		TypeInformation<User> te = TypeExtractor.createTypeInfo(User.class);

		assertEquals(AvroTypeInfo.class, te.getClass());
		TypeSerializer<User> tser = te.createSerializer(ec);

		ByteArrayOutputStream out = new ByteArrayOutputStream();
		try (DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out)) {
			tser.serialize(rec, outView);
		}

		User newRec;
		try (DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(
				new ByteArrayInputStream(out.toByteArray()))) {
			newRec = tser.deserialize(inView);
		}

		// check if it is still the same
		assertNotNull(newRec);
		assertEquals("name not equal", TEST_NAME, newRec.getName().toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), newRec.getTypeEnum().toString());
	}
}
 

/**
 * Creates a hint for the generic type in the class signature.
 */
public TypeHint() {
	try {
		this.typeInfo = TypeExtractor.createTypeInfo(
				this, TypeHint.class, getClass(), 0);
	}
	catch (InvalidTypesException e) {
		throw new FlinkRuntimeException("The TypeHint is using a generic variable." +
				"This is not supported, generic types must be fully specified for the TypeHint.");
	}
}
 

@Override
public TypeInformation<MaxWithRetractAccumulator<T>> getAccumulatorType() {
	PojoTypeInfo pojoType = (PojoTypeInfo) TypeExtractor.createTypeInfo(MaxWithRetractAccumulator.class);
	List<PojoField> pojoFields = new ArrayList<>();
	for (int i = 0; i < pojoType.getTotalFields(); i++) {
		PojoField field = pojoType.getPojoFieldAt(i);
		if (field.getField().getName().equals("max")) {
			pojoFields.add(new PojoField(field.getField(), getValueTypeInfo()));
		} else {
			pojoFields.add(field);
		}
	}
	//noinspection unchecked
	return new PojoTypeInfo(pojoType.getTypeClass(), pojoFields);
}
 

@Override
public TypeInformation<T> getProducedType() {
	if (this.getParser() instanceof ResultTypeQueryable) {
		return ((ResultTypeQueryable) this.getParser()).getProducedType();
	} else {
		return TypeExtractor.createTypeInfo(
			RowRecordParser.class,
			this.getParser().getClass(), 0, null, null);
	}
}
 

@Override
public TypeInformation<Row> getProducedType()
{
    TypeInformation<?>[] types = new TypeInformation<?>[] {
            TypeExtractor.createTypeInfo(String.class),
            TypeExtractor.createTypeInfo(String.class), //createTypeInformation[String]
            TypeExtractor.createTypeInfo(String.class),
            Types.INT,
            Types.LONG
    };
    return new RowTypeInfo(types, KAFKA_COLUMNS);
}
 

@Test
public void testRowSerializerWithComplexTypes() {
	RowTypeInfo typeInfo = new RowTypeInfo(
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.DOUBLE_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO,
		new TupleTypeInfo<Tuple3<Integer, Boolean, Short>>(
			BasicTypeInfo.INT_TYPE_INFO,
			BasicTypeInfo.BOOLEAN_TYPE_INFO,
			BasicTypeInfo.SHORT_TYPE_INFO),
		TypeExtractor.createTypeInfo(MyPojo.class));

	MyPojo testPojo1 = new MyPojo();
	testPojo1.name = null;
	MyPojo testPojo2 = new MyPojo();
	testPojo2.name = "Test1";
	MyPojo testPojo3 = new MyPojo();
	testPojo3.name = "Test2";

	Row[] data = new Row[]{
		createRow(null, null, null, null, null),
		createRow(0, null, null, null, null),
		createRow(0, 0.0, null, null, null),
		createRow(0, 0.0, "a", null, null),
		createRow(1, 0.0, "a", null, null),
		createRow(1, 1.0, "a", null, null),
		createRow(1, 1.0, "b", null, null),
		createRow(1, 1.0, "b", new Tuple3<>(1, false, (short) 2), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, false, (short) 2), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 2), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo1),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo2),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo3)
	};

	TypeSerializer<Row> serializer = typeInfo.createLegacySerializer(new ExecutionConfig());
	RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, data);
	testInstance.testAll();
}
 

@Test
public void testAtomicType() {
    StreamSchema<String> schema = new StreamSchema<>(TypeExtractor.createTypeInfo(String.class),
            "word");
    StreamSerializer<String> reader = new StreamSerializer<>(schema);
    Assert.assertArrayEquals(new Object[]{"Siddhi"}, reader.getRow("Siddhi"));
}
 

@Override
public TypeInformation<Tuple2<K, V>> getProducedType() {
	return new TupleTypeInfo<Tuple2<K, V>>(TypeExtractor.createTypeInfo(keyClass), TypeExtractor.createTypeInfo(valueClass));
}
 

@Test
public void testSerializerTree() {
	@SuppressWarnings("unchecked")
	TypeInformation<CollectionDataSets.PojoWithCollectionGeneric> ti =
			(TypeInformation<CollectionDataSets.PojoWithCollectionGeneric>)
					TypeExtractor.createTypeInfo(CollectionDataSets.PojoWithCollectionGeneric.class);

	String serTree = Utils.getSerializerTree(ti);
	// We can not test against the entire output because the fields of 'String' differ
	// between java versions
	Assert.assertTrue(serTree.startsWith("GenericTypeInfo (PojoWithCollectionGeneric)\n" +
			"    pojos:java.util.List\n" +
			"    key:int\n" +
			"    sqlDate:java.sql.Date\n" +
			"    bigInt:java.math.BigInteger\n" +
			"        signum:int\n" +
			"        mag:[I\n" +
			"        bitCount:int\n" +
			"        bitLength:int\n" +
			"        lowestSetBit:int\n" +
			"        firstNonzeroIntNum:int\n" +
			"    bigDecimalKeepItNull:java.math.BigDecimal\n" +
			"        intVal:java.math.BigInteger\n" +
			"            signum:int\n" +
			"            mag:[I\n" +
			"            bitCount:int\n" +
			"            bitLength:int\n" +
			"            lowestSetBit:int\n" +
			"            firstNonzeroIntNum:int\n" +
			"        scale:int\n" +
			"    scalaBigInt:scala.math.BigInt\n" +
			"        bigInteger:java.math.BigInteger\n" +
			"            signum:int\n" +
			"            mag:[I\n" +
			"            bitCount:int\n" +
			"            bitLength:int\n" +
			"            lowestSetBit:int\n" +
			"            firstNonzeroIntNum:int\n" +
			"    mixed:java.util.List\n" +
			"    makeMeGeneric:org.apache.flink.test.operators.util.CollectionDataSets$PojoWithDateAndEnum\n" +
			"        group:java.lang.String\n"));
}
 

@Override
public TypeInformation<T> getProducedType() {
	return TypeExtractor.createTypeInfo(NeighborsFunction.class, function.getClass(), 3, null, null);
}
 

public FlinkMapperWrapper(DoFn<KV<KI, VI>, KV<KO, VO>> fn) {
	this.fn = fn;
	
	Type out = ((ParameterizedType) fn.getClass().getGenericSuperclass()).getActualTypeArguments()[1];
	Type kot = ((ParameterizedType) out).getActualTypeArguments()[0];
	Type vot = ((ParameterizedType) out).getActualTypeArguments()[1];
	
	
	TypeInformation<KO> koi = (TypeInformation<KO>) TypeExtractor.createTypeInfo(kot);
	TypeInformation<VO> voi = (TypeInformation<VO>) TypeExtractor.createTypeInfo(vot);
	
	typeInformation = new TupleTypeInfo<>(koi, voi);
}
 

/**
 * Returns type information for a POJO (Plain Old Java Object).
 *
 * <p>A POJO class is public and standalone (no non-static inner class). It has a public no-argument
 * constructor. All non-static, non-transient fields in the class (and all superclasses) are either public
 * (and non-final) or have a public getter and a setter method that follows the Java beans naming
 * conventions for getters and setters.
 *
 * <p>A POJO is a fixed-length and null-aware composite type. Every field can be null independent
 * of the field's type.
 *
 * <p>The generic types for all fields of the POJO can be defined in a hierarchy of subclasses.
 *
 * <p>If Flink's type analyzer is unable to extract a valid POJO type information with
 * type information for all fields, an {@link org.apache.flink.api.common.functions.InvalidTypesException}
 * is thrown. Alternatively, you can use {@link Types#POJO(Class, Map)} to specify all fields manually.
 *
 * @param pojoClass POJO class to be analyzed by Flink
 */
public static <T> TypeInformation<T> POJO(Class<T> pojoClass) {
	final TypeInformation<T> ti = TypeExtractor.createTypeInfo(pojoClass);
	if (ti instanceof PojoTypeInfo) {
		return ti;
	}
	throw new InvalidTypesException("POJO type expected but was: " + ti);
}