Java Code Examples for org.apache.flink.api.common.ExecutionConfig#enableForceKryo()

@Benchmark
@OperationsPerInvocation(value = SerializationFrameworkMiniBenchmarks.RECORDS_PER_INVOCATION)
public void serializerKryoThrift(FlinkEnvironmentContext context) throws Exception {
	StreamExecutionEnvironment env = context.env;
	env.setParallelism(4);
	ExecutionConfig executionConfig = env.getConfig();
	executionConfig.enableForceKryo();
	executionConfig.addDefaultKryoSerializer(org.apache.flink.benchmark.thrift.MyPojo.class, TBaseSerializer.class);
	executionConfig.addDefaultKryoSerializer(org.apache.flink.benchmark.thrift.MyOperation.class, TBaseSerializer.class);

	env.addSource(new ThriftPojoSource(RECORDS_PER_INVOCATION, 10))
			.rebalance()
			.addSink(new DiscardingSink<>());

	env.execute();
}
 

@Benchmark
@OperationsPerInvocation(value = SerializationFrameworkMiniBenchmarks.RECORDS_PER_INVOCATION)
public void serializerKryoProtobuf(FlinkEnvironmentContext context) throws Exception {
	StreamExecutionEnvironment env = context.env;
	env.setParallelism(4);
	ExecutionConfig executionConfig = env.getConfig();
	executionConfig.enableForceKryo();
	executionConfig.registerTypeWithKryoSerializer(org.apache.flink.benchmark.protobuf.MyPojoOuterClass.MyPojo.class, ProtobufSerializer.class);
	executionConfig.registerTypeWithKryoSerializer(org.apache.flink.benchmark.protobuf.MyPojoOuterClass.MyOperation.class, ProtobufSerializer.class);

	env.addSource(new ProtobufPojoSource(RECORDS_PER_INVOCATION, 10))
			.rebalance()
			.addSink(new DiscardingSink<>());

	env.execute();
}
 

@Benchmark
@OperationsPerInvocation(value = SerializationFrameworkMiniBenchmarks.RECORDS_PER_INVOCATION)
public void serializerKryo(FlinkEnvironmentContext context) throws Exception {
	StreamExecutionEnvironment env = context.env;
	env.setParallelism(4);
	ExecutionConfig executionConfig = env.getConfig();
	executionConfig.enableForceKryo();
	executionConfig.registerKryoType(MyPojo.class);
	executionConfig.registerKryoType(MyOperation.class);

	env.addSource(new PojoSource(RECORDS_PER_INVOCATION, 10))
			.rebalance()
			.addSink(new DiscardingSink<>());

	env.execute();
}
 

@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final BaseMapSerializer serializer = createSerializerWithConfig(config);

	int inputKey = 998244353;
	MyObj inputObj = new MyObj(114514, 1919810);
	Map<Object, Object> javaMap = new HashMap<>();
	javaMap.put(inputKey, new BinaryGeneric<>(inputObj, new KryoSerializer<>(MyObj.class, config)));
	BaseMap inputMap = new GenericMap(javaMap);

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputMap, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final BaseMapSerializer restoreSerializer =
		(BaseMapSerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	BaseMap outputMap;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputMap = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreKeySer = restoreSerializer.getKeySerializer();
	TypeSerializer restoreValueSer = restoreSerializer.getValueSerializer();
	assertEquals(serializer.getKeySerializer(), restoreKeySer);
	assertEquals(serializer.getValueSerializer(), restoreValueSer);

	MyObj outputObj = BinaryGeneric.getJavaObjectFromBinaryGeneric(
		(BinaryGeneric) outputMap.toJavaMap(
			DataTypes.INT().getLogicalType(), DataTypes.ANY(TypeInformation.of(MyObj.class)).getLogicalType())
			.get(inputKey), new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}
 

@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final BaseArraySerializer serializer = createSerializerWithConfig(config);

	MyObj inputObj = new MyObj(114514, 1919810);
	BaseArray inputArray = new GenericArray(new BinaryGeneric[] {
		new BinaryGeneric<>(inputObj, new KryoSerializer<>(MyObj.class, config))
	}, 1);

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputArray, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final BaseArraySerializer restoreSerializer =
		(BaseArraySerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	BaseArray outputArray;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputArray = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreEleSer = restoreSerializer.getEleSer();
	assertEquals(serializer.getEleSer(), restoreEleSer);

	MyObj outputObj = BinaryGeneric.getJavaObjectFromBinaryGeneric(
		outputArray.getGeneric(0), new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}
 

@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final ArrayDataSerializer serializer = createSerializerWithConfig(config);

	MyObj inputObj = new MyObj(114514, 1919810);
	ArrayData inputArray = new GenericArrayData(new RawValueData[] {
		RawValueData.fromObject(inputObj)
	});

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputArray, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final ArrayDataSerializer restoreSerializer =
		(ArrayDataSerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	ArrayData outputArray;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputArray = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreEleSer = restoreSerializer.getEleSer();
	assertEquals(serializer.getEleSer(), restoreEleSer);

	MyObj outputObj = outputArray
		.<MyObj>getRawValue(0)
		.toObject(new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}
 

@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final MapDataSerializer serializer = createSerializerWithConfig(config);

	int inputKey = 998244353;
	MyObj inputObj = new MyObj(114514, 1919810);
	Map<Object, Object> javaMap = new HashMap<>();
	javaMap.put(inputKey, RawValueData.fromObject(inputObj));
	MapData inputMap = new GenericMapData(javaMap);

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputMap, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final MapDataSerializer restoreSerializer =
		(MapDataSerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	MapData outputMap;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputMap = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreKeySer = restoreSerializer.getKeySerializer();
	TypeSerializer restoreValueSer = restoreSerializer.getValueSerializer();
	assertEquals(serializer.getKeySerializer(), restoreKeySer);
	assertEquals(serializer.getValueSerializer(), restoreValueSer);

	Map<Object, Object> outputJavaMap = convertToJavaMap(
		outputMap,
		DataTypes.INT().getLogicalType(),
		DataTypes.RAW(TypeInformation.of(MyObj.class)).getLogicalType());
	MyObj outputObj = ((RawValueData<MyObj>) outputJavaMap.get(inputKey))
		.toObject(new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}