Java Code Examples for org.apache.flink.api.common.typeinfo.TypeInformation#createSerializer()

public CollectStreamResult(
		TableSchema tableSchema,
		ExecutionConfig config,
		InetAddress gatewayAddress,
		int gatewayPort,
		ClassLoader classLoader) {
	resultLock = new Object();

	// create socket stream iterator
	final TypeInformation<Tuple2<Boolean, Row>> socketType = Types.TUPLE(Types.BOOLEAN, tableSchema.toRowType());
	final TypeSerializer<Tuple2<Boolean, Row>> serializer = socketType.createSerializer(config);
	try {
		// pass gateway port and address such that iterator knows where to bind to
		iterator = new SocketStreamIterator<>(gatewayPort, gatewayAddress, serializer);
	} catch (IOException e) {
		throw new SqlClientException("Could not start socket for result retrieval.", e);
	}

	// create table sink
	// pass binding address and port such that sink knows where to send to
	collectTableSink = new CollectStreamTableSink(iterator.getBindAddress(), iterator.getPort(), serializer, tableSchema);
	retrievalThread = new ResultRetrievalThread();

	this.classLoader = checkNotNull(classLoader);
}
 

RecursiveProductFieldAccessor(int pos, TypeInformation<T> typeInfo, FieldAccessor<R, F> innerAccessor, ExecutionConfig config) {
	int arity = ((TupleTypeInfoBase) typeInfo).getArity();
	if (pos < 0 || pos >= arity) {
		throw new CompositeType.InvalidFieldReferenceException(
			"Tried to select " + ((Integer) pos).toString() + ". field on \"" +
				typeInfo.toString() + "\", which is an invalid index.");
	}
	checkNotNull(typeInfo, "typeInfo must not be null.");
	checkNotNull(innerAccessor, "innerAccessor must not be null.");

	this.pos = pos;
	this.fieldType = ((TupleTypeInfoBase<T>) typeInfo).getTypeAt(pos);
	this.serializer = (TupleSerializerBase<T>) typeInfo.createSerializer(config);
	this.length = this.serializer.getArity();
	this.fields = new Object[this.length];
	this.innerAccessor = innerAccessor;
}
 

@Test
public void testLargeRowSerializer() {
	TypeInformation<Row> typeInfo = new RowTypeInfo(
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO);

	Row row = new Row(13);
	row.setField(0, 2);
	row.setField(1, null);
	row.setField(3, null);
	row.setField(4, null);
	row.setField(5, null);
	row.setField(6, null);
	row.setField(7, null);
	row.setField(8, null);
	row.setField(9, null);
	row.setField(10, null);
	row.setField(11, null);
	row.setField(12, "Test");

	TypeSerializer<Row> serializer = typeInfo.createSerializer(new ExecutionConfig());
	RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, row);
	testInstance.testAll();
}
 

@Test
public void testStringArray() {
	String[][] array = new String[][]{{null,"b"},{"c","d"},{"e","f"},{"g","h"},null};
	TypeInformation<String[][]> ti = TypeExtractor.getForClass(String[][].class);

	SerializerTestInstance<String[][]> testInstance = new SerializerTestInstance<String[][]>(ti.createSerializer(new ExecutionConfig()), String[][].class, -1, array);
	testInstance.testAll();
}
 

public <IN1, IN2, OUT> void addCoOperator(
		Integer vertexID,
		String slotSharingGroup,
		@Nullable String coLocationGroup,
		StreamOperatorFactory<OUT> taskOperatorFactory,
		TypeInformation<IN1> in1TypeInfo,
		TypeInformation<IN2> in2TypeInfo,
		TypeInformation<OUT> outTypeInfo,
		String operatorName) {

	Class<? extends AbstractInvokable> vertexClass = taskOperatorFactory.isOperatorSelectiveReading() ?
		TwoInputSelectableStreamTask.class : TwoInputStreamTask.class;

	addNode(vertexID, slotSharingGroup, coLocationGroup, vertexClass, taskOperatorFactory, operatorName);

	TypeSerializer<OUT> outSerializer = (outTypeInfo != null) && !(outTypeInfo instanceof MissingTypeInfo) ?
			outTypeInfo.createSerializer(executionConfig) : null;

	setSerializers(vertexID, in1TypeInfo.createSerializer(executionConfig), in2TypeInfo.createSerializer(executionConfig), outSerializer);

	if (taskOperatorFactory.isOutputTypeConfigurable()) {
		// sets the output type which must be know at StreamGraph creation time
		taskOperatorFactory.setOutputType(outTypeInfo, executionConfig);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("CO-TASK: {}", vertexID);
	}
}
 

@Override
protected <T> TypeSerializer<T> createSerializer(Class<T> type) {
	ExecutionConfig conf = new ExecutionConfig();
	conf.registerTypeWithKryoSerializer(LocalDate.class, LocalDateSerializer.class);
	TypeInformation<T> typeInfo = new GenericTypeInfo<T>(type);
	return typeInfo.createSerializer(conf);
}
 

@Test
public void testObjectArrays() {
	Integer[][] array = new Integer[][]{{0,1}, null, {null, 42}};
	TypeInformation<Integer[][]> ti = TypeExtractor.getForClass(Integer[][].class);

	SerializerTestInstance<Integer[][]> testInstance = new SerializerTestInstance<Integer[][]>(ti.createSerializer(new ExecutionConfig()), Integer[][].class, -1, array);
	testInstance.testAll();

	MyPojo[][] array2 = new MyPojo[][]{{new MyPojo(null, 42), new MyPojo("test2", -1)}, {null, null}, null};
	TypeInformation<MyPojo[][]> ti2 = TypeExtractor.getForClass(MyPojo[][].class);

	SerializerTestInstance<MyPojo[][]> testInstance2 = new SerializerTestInstance<MyPojo[][]>(ti2.createSerializer(new ExecutionConfig()), MyPojo[][].class, -1, array2);
	testInstance2.testAll();
}
 

@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testGenericObjectArrays() {
	MyGenericPojo<String>[][] array = (MyGenericPojo<String>[][]) new MyGenericPojo[][]{
		{ new MyGenericPojo<String>(new String[][]{{"a", "b"},{"c", "d"}}), null}
	};

	TypeInformation<MyGenericPojo<String>[][]> ti =
			TypeInformation.of(new TypeHint<MyGenericPojo<String>[][]>(){});

	SerializerTestInstance testInstance = new SerializerTestInstance(ti.createSerializer(new ExecutionConfig()), MyGenericPojo[][].class, -1, (Object) array);
	testInstance.testAll();
}
 

@Override
protected List<OUT> executeOnCollections(List<IN1> leftInput, List<IN2> rightInput, RuntimeContext runtimeContext, ExecutionConfig executionConfig) throws Exception {
	TypeInformation<IN1> leftInformation = getOperatorInfo().getFirstInputType();
	TypeInformation<IN2> rightInformation = getOperatorInfo().getSecondInputType();
	TypeInformation<OUT> outInformation = getOperatorInfo().getOutputType();

	TypeComparator<IN1> leftComparator = buildComparatorFor(0, executionConfig, leftInformation);
	TypeComparator<IN2> rightComparator = buildComparatorFor(1, executionConfig, rightInformation);

	TypeSerializer<IN1> leftSerializer = leftInformation.createSerializer(executionConfig);
	TypeSerializer<IN2> rightSerializer = rightInformation.createSerializer(executionConfig);

	OuterJoinListIterator<IN1, IN2> outerJoinIterator =
			new OuterJoinListIterator<>(leftInput, leftSerializer, leftComparator,
					rightInput, rightSerializer, rightComparator, outerJoinType);

	// --------------------------------------------------------------------
	// Run UDF
	// --------------------------------------------------------------------
	FlatJoinFunction<IN1, IN2, OUT> function = userFunction.getUserCodeObject();

	FunctionUtils.setFunctionRuntimeContext(function, runtimeContext);
	FunctionUtils.openFunction(function, this.parameters);

	List<OUT> result = new ArrayList<>();
	Collector<OUT> collector = new CopyingListCollector<>(result, outInformation.createSerializer(executionConfig));

	while (outerJoinIterator.next()) {
		IN1 left = outerJoinIterator.getLeft();
		IN2 right = outerJoinIterator.getRight();
		function.join(left == null ? null : leftSerializer.copy(left), right == null ? null : rightSerializer.copy(right), collector);
	}

	FunctionUtils.closeFunction(function);

	return result;
}
 

@Override
public <S extends Serializable> ValueState<S> getKeyValueState(String name,
	TypeInformation<S> stateType,
	S defaultState) {

	checkNotNull(name, "The name of the state must not be null");
	checkNotNull(stateType, "The state type information must not be null");

	ValueStateDescriptor<S> stateDesc = new ValueStateDescriptor<>(name, stateType.createSerializer(getExecutionConfig()), defaultState);
	return getPartitionedState(stateDesc);
}
 

@Test
public void testSerializability() {
	try (ByteArrayOutputStream buffer = new ByteArrayOutputStream();
		ObjectOutputStream out = new ObjectOutputStream(buffer)) {

		Collection<ElementType> inputCollection = new ArrayList<ElementType>();
		ElementType element1 = new ElementType(1);
		ElementType element2 = new ElementType(2);
		ElementType element3 = new ElementType(3);
		inputCollection.add(element1);
		inputCollection.add(element2);
		inputCollection.add(element3);

		@SuppressWarnings("unchecked")
		TypeInformation<ElementType> info = (TypeInformation<ElementType>) TypeExtractor.createTypeInfo(ElementType.class);

		CollectionInputFormat<ElementType> inputFormat = new CollectionInputFormat<ElementType>(inputCollection,
				info.createSerializer(new ExecutionConfig()));

		out.writeObject(inputFormat);

		ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(buffer.toByteArray()));

		Object serializationResult = in.readObject();

		assertNotNull(serializationResult);
		assertTrue(serializationResult instanceof CollectionInputFormat<?>);

		@SuppressWarnings("unchecked")
		CollectionInputFormat<ElementType> result = (CollectionInputFormat<ElementType>) serializationResult;

		GenericInputSplit inputSplit = new GenericInputSplit(0, 1);
		inputFormat.open(inputSplit);
		result.open(inputSplit);

		while (!inputFormat.reachedEnd() && !result.reachedEnd()){
			ElementType expectedElement = inputFormat.nextRecord(null);
			ElementType actualElement = result.nextRecord(null);

			assertEquals(expectedElement, actualElement);
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.toString());
	}

}
 

@Override
@SuppressWarnings("unchecked")
public void setInputType(TypeInformation<?> type, ExecutionConfig executionConfig) {
	serializer = (TypeSerializer<T>) type.createSerializer(executionConfig);
}
 

@Override
protected List<OUT> executeOnCollections(List<IN1> input1, List<IN2> input2, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	// --------------------------------------------------------------------
	// Setup
	// --------------------------------------------------------------------
	TypeInformation<IN1> inputType1 = getOperatorInfo().getFirstInputType();
	TypeInformation<IN2> inputType2 = getOperatorInfo().getSecondInputType();

	int[] inputKeys1 = getKeyColumns(0);
	int[] inputKeys2 = getKeyColumns(1);

	boolean[] inputSortDirections1 = new boolean[inputKeys1.length];
	boolean[] inputSortDirections2 = new boolean[inputKeys2.length];

	Arrays.fill(inputSortDirections1, true);
	Arrays.fill(inputSortDirections2, true);

	final TypeSerializer<IN1> inputSerializer1 = inputType1.createSerializer(executionConfig);
	final TypeSerializer<IN2> inputSerializer2 = inputType2.createSerializer(executionConfig);

	final TypeComparator<IN1> inputComparator1 = getTypeComparator(executionConfig, inputType1, inputKeys1, inputSortDirections1);
	final TypeComparator<IN2> inputComparator2 = getTypeComparator(executionConfig, inputType2, inputKeys2, inputSortDirections2);

	SimpleListIterable<IN1> iterator1 = new SimpleListIterable<IN1>(input1, inputComparator1, inputSerializer1);
	SimpleListIterable<IN2> iterator2 = new SimpleListIterable<IN2>(input2, inputComparator2, inputSerializer2);

	// --------------------------------------------------------------------
	// Run UDF
	// --------------------------------------------------------------------
	CoGroupFunction<IN1, IN2, OUT> function = userFunction.getUserCodeObject();

	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, parameters);

	List<OUT> result = new ArrayList<OUT>();
	Collector<OUT> resultCollector = new CopyingListCollector<OUT>(result, getOperatorInfo().getOutputType().createSerializer(executionConfig));

	function.coGroup(iterator1, iterator2, resultCollector);

	FunctionUtils.closeFunction(function);

	return result;
}
 

@Test
public void testRestoreKryoSerializedKeysWindows() throws Exception {
	final int windowSize = 3;

	TypeInformation<Tuple2<NonPojoType, Integer>> inputType = new TypeHint<Tuple2<NonPojoType, Integer>>() {}.getTypeInfo();

	ReducingStateDescriptor<Tuple2<NonPojoType, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer<>(),
		inputType.createSerializer(new ExecutionConfig()));

	TypeSerializer<NonPojoType> keySerializer = TypeInformation.of(NonPojoType.class).createSerializer(new ExecutionConfig());
	assertTrue(keySerializer instanceof KryoSerializer);

	WindowOperator<NonPojoType, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, TimeWindow> operator = new WindowOperator<>(
		TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
		new TimeWindow.Serializer(),
		new TupleKeySelector<>(),
		keySerializer,
		stateDesc,
		new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<>()),
		EventTimeTrigger.create(),
		0,
		null /* late data output tag */);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	OneInputStreamOperatorTestHarness<Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), TypeInformation.of(NonPojoType.class));

	testHarness.setup();

	testHarness.initializeState(
		OperatorSnapshotUtil.getResourceFilename(
			"win-op-migration-test-kryo-serialized-key-flink" + testMigrateVersion + "-snapshot"));

	testHarness.open();

	testHarness.processWatermark(new Watermark(2999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 3), 2999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 3), 2999));
	expectedOutput.add(new Watermark(2999));

	testHarness.processWatermark(new Watermark(3999));
	expectedOutput.add(new Watermark(3999));

	testHarness.processWatermark(new Watermark(4999));
	expectedOutput.add(new Watermark(4999));

	testHarness.processWatermark(new Watermark(5999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 2), 5999));
	expectedOutput.add(new Watermark(5999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
	testHarness.close();
}
 

@Override
public void setOutputType(TypeInformation<OUT> outTypeInfo, ExecutionConfig executionConfig) {
	this.serializer = outTypeInfo.createSerializer(executionConfig);
}
 

@Override
public void setOutputType(TypeInformation<OUT> outTypeInfo, ExecutionConfig executionConfig) {
	this.serializer = outTypeInfo.createSerializer(executionConfig);
}
 

/**
 * Manually run this to write binary snapshot data.
 */
@Ignore
@Test
public void writeWindowsWithKryoSerializedKeysSnapshot() throws Exception {
	final int windowSize = 3;

	TypeInformation<Tuple2<NonPojoType, Integer>> inputType = new TypeHint<Tuple2<NonPojoType, Integer>>() {}.getTypeInfo();

	ReducingStateDescriptor<Tuple2<NonPojoType, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
		new SumReducer<>(),
		inputType.createSerializer(new ExecutionConfig()));

	TypeSerializer<NonPojoType> keySerializer = TypeInformation.of(NonPojoType.class).createSerializer(new ExecutionConfig());
	assertTrue(keySerializer instanceof KryoSerializer);

	WindowOperator<NonPojoType, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, TimeWindow> operator = new WindowOperator<>(
		TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
		new TimeWindow.Serializer(),
		new TupleKeySelector<>(),
		keySerializer,
		stateDesc,
		new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<>()),
		EventTimeTrigger.create(),
		0,
		null /* late data output tag */);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	OneInputStreamOperatorTestHarness<Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), TypeInformation.of(NonPojoType.class));

	testHarness.setup();
	testHarness.open();

	// add elements out-of-order
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 3999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 3000));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 20));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 0));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1998));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1000));

	testHarness.processWatermark(new Watermark(999));
	expectedOutput.add(new Watermark(999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());

	testHarness.processWatermark(new Watermark(1999));
	expectedOutput.add(new Watermark(1999));
	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());

	// do snapshot and save to file
	OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);
	OperatorSnapshotUtil.writeStateHandle(
		snapshot,
		"src/test/resources/win-op-migration-test-kryo-serialized-key-flink" + flinkGenerateSavepointVersion + "-snapshot");

	testHarness.close();
}
 

public static TypeSerializer<Event> createTypeSerializer() {
	TypeInformation<Event> typeInformation = (TypeInformation<Event>) TypeExtractor.createTypeInfo(Event.class);

	return typeInformation.createSerializer(new ExecutionConfig());
}
 

/**
 * Creates a new de-/serialization schema for the given type.
 *
 * @param typeInfo The type information for the type de-/serialized by this schema.
 * @param ec The execution config, which is used to parametrize the type serializers.
 */
public TypeInformationSerializationSchema(TypeInformation<T> typeInfo, ExecutionConfig ec) {
	this.typeInfo = checkNotNull(typeInfo, "typeInfo");
	this.serializer = typeInfo.createSerializer(ec);
}
 

/**
 * Creates a new de-/serialization schema for the given type.
 *
 * @param typeInfo The type information for the type de-/serialized by this schema.
 * @param ec The execution config, which is used to parametrize the type serializers.
 */
public TypeInformationSerializationSchema(TypeInformation<T> typeInfo, ExecutionConfig ec) {
	this.typeInfo = checkNotNull(typeInfo, "typeInfo");
	this.serializer = typeInfo.createSerializer(ec);
}