Java Code Examples for org.apache.flink.api.common.ExecutionConfig#registerTypeWithKryoSerializer()

@Override
public void addAvroSerializersIfRequired(ExecutionConfig reg, Class<?> type) {
	if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(type) ||
		org.apache.avro.generic.GenericData.Record.class.isAssignableFrom(type)) {

		// Avro POJOs contain java.util.List which have GenericData.Array as their runtime type
		// because Kryo is not able to serialize them properly, we use this serializer for them
		reg.registerTypeWithKryoSerializer(GenericData.Array.class, Serializers.SpecificInstanceCollectionSerializerForArrayList.class);

		// We register this serializer for users who want to use untyped Avro records (GenericData.Record).
		// Kryo is able to serialize everything in there, except for the Schema.
		// This serializer is very slow, but using the GenericData.Records of Kryo is in general a bad idea.
		// we add the serializer as a default serializer because Avro is using a private sub-type at runtime.
		reg.addDefaultKryoSerializer(Schema.class, AvroSchemaSerializer.class);
	}
}
 

@Override
public void addAvroSerializersIfRequired(ExecutionConfig reg, Class<?> type) {
	if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(type) ||
		org.apache.avro.generic.GenericData.Record.class.isAssignableFrom(type)) {

		// Avro POJOs contain java.util.List which have GenericData.Array as their runtime type
		// because Kryo is not able to serialize them properly, we use this serializer for them
		reg.registerTypeWithKryoSerializer(GenericData.Array.class, Serializers.SpecificInstanceCollectionSerializerForArrayList.class);

		// We register this serializer for users who want to use untyped Avro records (GenericData.Record).
		// Kryo is able to serialize everything in there, except for the Schema.
		// This serializer is very slow, but using the GenericData.Records of Kryo is in general a bad idea.
		// we add the serializer as a default serializer because Avro is using a private sub-type at runtime.
		reg.addDefaultKryoSerializer(Schema.class, AvroSchemaSerializer.class);
	}
}
 

public static void registerSerializers(ExecutionConfig conf) {

        Class<GuavaImmutableMapSerializer> serializer = GuavaImmutableMapSerializer.class;

        conf.registerTypeWithKryoSerializer(ImmutableMap.class, serializer);
        conf.registerTypeWithKryoSerializer(ImmutableMap.of().getClass(), serializer);
        Object o1 = new Object();
        Object o2 = new Object();
        conf.registerTypeWithKryoSerializer(ImmutableMap.of(o1, o1).getClass(), serializer);
        conf.registerTypeWithKryoSerializer(ImmutableMap.of(o1, o1, o2, o2).getClass(), serializer);
        Map<DummyEnum, Object> enumMap = new EnumMap<>(DummyEnum.class);
        for (DummyEnum e : DummyEnum.values()) {
            enumMap.put(e, o1);
        }
        conf.registerTypeWithKryoSerializer(ImmutableMap.copyOf(enumMap).getClass(), serializer);
        ImmutableTable<Object, Object, Object> denseImmutableTable = ImmutableTable.builder().put("a", 1, 1).put("b", 1, 1).build();
        conf.registerTypeWithKryoSerializer(denseImmutableTable.rowMap().getClass(), serializer);
        conf.registerTypeWithKryoSerializer(((Map) denseImmutableTable.rowMap().get("a")).getClass(), serializer);
        conf.registerTypeWithKryoSerializer(denseImmutableTable.columnMap().getClass(), serializer);
        conf.registerTypeWithKryoSerializer(((Map) denseImmutableTable.columnMap().get(1)).getClass(), serializer);
    }
 

@Override
public TypeSerializer<Animal> createUpgradedSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerKryoType(DummyClassOne.class);
	executionConfig.registerTypeWithKryoSerializer(
			DummyClassTwo.class,
			DefaultSerializers.StringSerializer.class);

	return new KryoSerializer<>(Animal.class, executionConfig);
}
 

@Override
protected <T> TypeSerializer<T> createSerializer(Class<T> type) {
	ExecutionConfig conf = new ExecutionConfig();
	conf.registerTypeWithKryoSerializer(LocalDate.class, LocalDateSerializer.class);
	TypeInformation<T> typeInfo = new GenericTypeInfo<T>(type);
	return typeInfo.createSerializer(conf);
}
 

@Override
protected <T> TypeSerializer<T> createSerializer(Class<T> type) {
	ExecutionConfig conf = new ExecutionConfig();
	conf.registerTypeWithKryoSerializer(LocalDate.class, LocalDateSerializer.class);
	TypeInformation<T> typeInfo = new GenericTypeInfo<T>(type);
	return typeInfo.createSerializer(conf);
}
 

@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final BaseMapSerializer serializer = createSerializerWithConfig(config);

	int inputKey = 998244353;
	MyObj inputObj = new MyObj(114514, 1919810);
	Map<Object, Object> javaMap = new HashMap<>();
	javaMap.put(inputKey, new BinaryGeneric<>(inputObj, new KryoSerializer<>(MyObj.class, config)));
	BaseMap inputMap = new GenericMap(javaMap);

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputMap, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final BaseMapSerializer restoreSerializer =
		(BaseMapSerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	BaseMap outputMap;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputMap = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreKeySer = restoreSerializer.getKeySerializer();
	TypeSerializer restoreValueSer = restoreSerializer.getValueSerializer();
	assertEquals(serializer.getKeySerializer(), restoreKeySer);
	assertEquals(serializer.getValueSerializer(), restoreValueSer);

	MyObj outputObj = BinaryGeneric.getJavaObjectFromBinaryGeneric(
		(BinaryGeneric) outputMap.toJavaMap(
			DataTypes.INT().getLogicalType(), DataTypes.ANY(TypeInformation.of(MyObj.class)).getLogicalType())
			.get(inputKey), new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}
 

@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final BaseArraySerializer serializer = createSerializerWithConfig(config);

	MyObj inputObj = new MyObj(114514, 1919810);
	BaseArray inputArray = new GenericArray(new BinaryGeneric[] {
		new BinaryGeneric<>(inputObj, new KryoSerializer<>(MyObj.class, config))
	}, 1);

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputArray, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final BaseArraySerializer restoreSerializer =
		(BaseArraySerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	BaseArray outputArray;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputArray = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreEleSer = restoreSerializer.getEleSer();
	assertEquals(serializer.getEleSer(), restoreEleSer);

	MyObj outputObj = BinaryGeneric.getJavaObjectFromBinaryGeneric(
		outputArray.getGeneric(0), new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}
 

@Override
public TypeSerializer<Animal> createUpgradedSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerKryoType(DummyClassOne.class);
	executionConfig.registerTypeWithKryoSerializer(
			Dog.class,
			KryoPojosForMigrationTests.DogV2KryoSerializer.class);
	executionConfig.registerKryoType(DummyClassTwo.class);
	executionConfig.registerKryoType(Cat.class);
	executionConfig.registerTypeWithKryoSerializer(
			Parrot.class,
			KryoPojosForMigrationTests.ParrotKryoSerializer.class);

	return new KryoSerializer<>(Animal.class, executionConfig);
}
 

@Override
protected <T> TypeSerializer<T> createSerializer(Class<T> type) {
	ExecutionConfig conf = new ExecutionConfig();
	conf.registerTypeWithKryoSerializer(LocalDate.class, LocalDateSerializer.class);
	TypeInformation<T> typeInfo = new GenericTypeInfo<T>(type);
	return typeInfo.createSerializer(conf);
}
 

@Test
public void testDuplicateSerializerWithRegisteredSerializerInstance() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerTypeWithKryoSerializer(WrappedString.class, new TestSerializer());
	runDuplicateSerializerTest(executionConfig);
}
 

@Test
public void testExecutionConfigWithKryo() throws Exception {
	// serialize base array
	ExecutionConfig config = new ExecutionConfig();
	config.enableForceKryo();
	config.registerTypeWithKryoSerializer(MyObj.class, new MyObjSerializer());
	final MapDataSerializer serializer = createSerializerWithConfig(config);

	int inputKey = 998244353;
	MyObj inputObj = new MyObj(114514, 1919810);
	Map<Object, Object> javaMap = new HashMap<>();
	javaMap.put(inputKey, RawValueData.fromObject(inputObj));
	MapData inputMap = new GenericMapData(javaMap);

	byte[] serialized;
	try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
		serializer.serialize(inputMap, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	// deserialize base array using restored serializer
	final MapDataSerializer restoreSerializer =
		(MapDataSerializer) snapshotAndReconfigure(serializer, () -> createSerializerWithConfig(config));

	MapData outputMap;
	try (ByteArrayInputStream in = new ByteArrayInputStream(serialized)) {
		outputMap = restoreSerializer.deserialize(new DataInputViewStreamWrapper(in));
	}

	TypeSerializer restoreKeySer = restoreSerializer.getKeySerializer();
	TypeSerializer restoreValueSer = restoreSerializer.getValueSerializer();
	assertEquals(serializer.getKeySerializer(), restoreKeySer);
	assertEquals(serializer.getValueSerializer(), restoreValueSer);

	Map<Object, Object> outputJavaMap = convertToJavaMap(
		outputMap,
		DataTypes.INT().getLogicalType(),
		DataTypes.RAW(TypeInformation.of(MyObj.class)).getLogicalType());
	MyObj outputObj = ((RawValueData<MyObj>) outputJavaMap.get(inputKey))
		.toObject(new KryoSerializer<>(MyObj.class, config));
	assertEquals(inputObj, outputObj);
}
 

/**
 * Tests that the kryo output buffer is cleared in case of an exception. Flink uses the
 * EOFException to signal that a buffer is full. In such a case, the record which was tried
 * to be written will be rewritten. Therefore, eventually buffered data of this record has
 * to be cleared.
 */
@Test
public void testOutputBufferedBeingClearedInCaseOfException() throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerTypeWithKryoSerializer(TestRecord.class, new TestRecordSerializer());
	executionConfig.registerKryoType(TestRecord.class);

	KryoSerializer<TestRecord> kryoSerializer = new KryoSerializer<TestRecord>(
		TestRecord.class,
		executionConfig);

	int size = 94;
	int bufferSize = 150;

	TestRecord testRecord = new TestRecord(size);

	TestDataOutputView target = new TestDataOutputView(bufferSize);

	kryoSerializer.serialize(testRecord, target);

	try {
		kryoSerializer.serialize(testRecord, target);
		Assert.fail("Expected an EOFException.");
	} catch(EOFException eofException) {
		// expected exception
		// now the Kryo Output should have been cleared
	}

	TestRecord actualRecord = kryoSerializer.deserialize(
			new DataInputViewStreamWrapper(new ByteArrayInputStream(target.getBuffer())));

	Assert.assertEquals(testRecord, actualRecord);

	target.clear();

	// if the kryo output has been cleared then we can serialize our test record into the target
	// because the target buffer 150 bytes can host one TestRecord (total serialization size 100)
	kryoSerializer.serialize(testRecord, target);

	byte[] buffer = target.getBuffer();
	int counter = 0;

	for (int i = 0; i < buffer.length; i++) {
		if(buffer[i] == 42) {
			counter++;
		}
	}

	Assert.assertEquals(size, counter);
}
 

@Test
public void testRegisterStatesWithoutTypeSerializer() throws Exception {
	// prepare an execution config with a non standard type registered
	final Class<?> registeredType = FutureTask.class;

	// validate the precondition of this test - if this condition fails, we need to pick a different
	// example serializer
	assertFalse(new KryoSerializer<>(File.class, new ExecutionConfig()).getKryo().getDefaultSerializer(registeredType)
			instanceof com.esotericsoftware.kryo.serializers.JavaSerializer);

	final ExecutionConfig cfg = new ExecutionConfig();
	cfg.registerTypeWithKryoSerializer(registeredType, com.esotericsoftware.kryo.serializers.JavaSerializer.class);

	final OperatorStateBackend operatorStateBackend =
		new DefaultOperatorStateBackendBuilder(
			classLoader,
			cfg,
			false,
			emptyStateHandles,
			new CloseableRegistry())
		.build();

	ListStateDescriptor<File> stateDescriptor = new ListStateDescriptor<>("test", File.class);
	ListStateDescriptor<String> stateDescriptor2 = new ListStateDescriptor<>("test2", String.class);

	ListState<File> listState = operatorStateBackend.getListState(stateDescriptor);
	assertNotNull(listState);

	ListState<String> listState2 = operatorStateBackend.getListState(stateDescriptor2);
	assertNotNull(listState2);

	assertEquals(2, operatorStateBackend.getRegisteredStateNames().size());

	// make sure that type registrations are forwarded
	TypeSerializer<?> serializer = ((PartitionableListState<?>) listState).getStateMetaInfo().getPartitionStateSerializer();
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getSerializer(registeredType)
			instanceof com.esotericsoftware.kryo.serializers.JavaSerializer);

	Iterator<String> it = listState2.get().iterator();
	assertFalse(it.hasNext());
	listState2.add("kevin");
	listState2.add("sunny");

	it = listState2.get().iterator();
	assertEquals("kevin", it.next());
	assertEquals("sunny", it.next());
	assertFalse(it.hasNext());
}
 

@Test
public void testDuplicateSerializerWithRegisteredSerializerClass() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerTypeWithKryoSerializer(WrappedString.class, TestSerializer.class);
	runDuplicateSerializerTest(executionConfig);
}
 

/**
 * Tests that the kryo output buffer is cleared in case of an exception. Flink uses the
 * EOFException to signal that a buffer is full. In such a case, the record which was tried
 * to be written will be rewritten. Therefore, eventually buffered data of this record has
 * to be cleared.
 */
@Test
public void testOutputBufferedBeingClearedInCaseOfException() throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerTypeWithKryoSerializer(TestRecord.class, new TestRecordSerializer());
	executionConfig.registerKryoType(TestRecord.class);

	KryoSerializer<TestRecord> kryoSerializer = new KryoSerializer<TestRecord>(
		TestRecord.class,
		executionConfig);

	int size = 94;
	int bufferSize = 150;

	TestRecord testRecord = new TestRecord(size);

	TestDataOutputView target = new TestDataOutputView(bufferSize);

	kryoSerializer.serialize(testRecord, target);

	try {
		kryoSerializer.serialize(testRecord, target);
		Assert.fail("Expected an EOFException.");
	} catch(EOFException eofException) {
		// expected exception
		// now the Kryo Output should have been cleared
	}

	TestRecord actualRecord = kryoSerializer.deserialize(
			new DataInputViewStreamWrapper(new ByteArrayInputStream(target.getBuffer())));

	Assert.assertEquals(testRecord, actualRecord);

	target.clear();

	// if the kryo output has been cleared then we can serialize our test record into the target
	// because the target buffer 150 bytes can host one TestRecord (total serialization size 100)
	kryoSerializer.serialize(testRecord, target);

	byte[] buffer = target.getBuffer();
	int counter = 0;

	for (int i = 0; i < buffer.length; i++) {
		if(buffer[i] == 42) {
			counter++;
		}
	}

	Assert.assertEquals(size, counter);
}
 

@Test
public void testRegisterStatesWithoutTypeSerializer() throws Exception {
	// prepare an execution config with a non standard type registered
	final Class<?> registeredType = FutureTask.class;

	// validate the precondition of this test - if this condition fails, we need to pick a different
	// example serializer
	assertFalse(new KryoSerializer<>(File.class, new ExecutionConfig()).getKryo().getDefaultSerializer(registeredType)
			instanceof com.esotericsoftware.kryo.serializers.JavaSerializer);

	final ExecutionConfig cfg = new ExecutionConfig();
	cfg.registerTypeWithKryoSerializer(registeredType, com.esotericsoftware.kryo.serializers.JavaSerializer.class);

	final OperatorStateBackend operatorStateBackend =
		new DefaultOperatorStateBackendBuilder(
			classLoader,
			cfg,
			false,
			emptyStateHandles,
			new CloseableRegistry())
		.build();

	ListStateDescriptor<File> stateDescriptor = new ListStateDescriptor<>("test", File.class);
	ListStateDescriptor<String> stateDescriptor2 = new ListStateDescriptor<>("test2", String.class);

	ListState<File> listState = operatorStateBackend.getListState(stateDescriptor);
	assertNotNull(listState);

	ListState<String> listState2 = operatorStateBackend.getListState(stateDescriptor2);
	assertNotNull(listState2);

	assertEquals(2, operatorStateBackend.getRegisteredStateNames().size());

	// make sure that type registrations are forwarded
	TypeSerializer<?> serializer = ((PartitionableListState<?>) listState).getStateMetaInfo().getPartitionStateSerializer();
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getSerializer(registeredType)
			instanceof com.esotericsoftware.kryo.serializers.JavaSerializer);

	Iterator<String> it = listState2.get().iterator();
	assertFalse(it.hasNext());
	listState2.add("kevin");
	listState2.add("sunny");

	it = listState2.get().iterator();
	assertEquals("kevin", it.next());
	assertEquals("sunny", it.next());
	assertFalse(it.hasNext());
}
 

@Test
public void testRegisterStatesWithoutTypeSerializer() throws Exception {
	// prepare an execution config with a non standard type registered
	final Class<?> registeredType = FutureTask.class;

	// validate the precondition of this test - if this condition fails, we need to pick a different
	// example serializer
	assertFalse(new KryoSerializer<>(File.class, new ExecutionConfig()).getKryo().getDefaultSerializer(registeredType)
			instanceof com.esotericsoftware.kryo.serializers.JavaSerializer);

	final ExecutionConfig cfg = new ExecutionConfig();
	cfg.registerTypeWithKryoSerializer(registeredType, com.esotericsoftware.kryo.serializers.JavaSerializer.class);

	final OperatorStateBackend operatorStateBackend =
		new DefaultOperatorStateBackendBuilder(
			classLoader,
			cfg,
			false,
			emptyStateHandles,
			new CloseableRegistry())
		.build();

	ListStateDescriptor<File> stateDescriptor = new ListStateDescriptor<>("test", File.class);
	ListStateDescriptor<String> stateDescriptor2 = new ListStateDescriptor<>("test2", String.class);

	ListState<File> listState = operatorStateBackend.getListState(stateDescriptor);
	assertNotNull(listState);

	ListState<String> listState2 = operatorStateBackend.getListState(stateDescriptor2);
	assertNotNull(listState2);

	assertEquals(2, operatorStateBackend.getRegisteredStateNames().size());

	// make sure that type registrations are forwarded
	TypeSerializer<?> serializer = ((PartitionableListState<?>) listState).getStateMetaInfo().getPartitionStateSerializer();
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getSerializer(registeredType)
			instanceof com.esotericsoftware.kryo.serializers.JavaSerializer);

	Iterator<String> it = listState2.get().iterator();
	assertFalse(it.hasNext());
	listState2.add("kevin");
	listState2.add("sunny");

	it = listState2.get().iterator();
	assertEquals("kevin", it.next());
	assertEquals("sunny", it.next());
	assertFalse(it.hasNext());
}
 

@Test
public void testDuplicateSerializerWithRegisteredSerializerClass() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerTypeWithKryoSerializer(WrappedString.class, TestSerializer.class);
	runDuplicateSerializerTest(executionConfig);
}
 

/**
 * Tests that the kryo output buffer is cleared in case of an exception. Flink uses the
 * EOFException to signal that a buffer is full. In such a case, the record which was tried
 * to be written will be rewritten. Therefore, eventually buffered data of this record has
 * to be cleared.
 */
@Test
public void testOutputBufferedBeingClearedInCaseOfException() throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.registerTypeWithKryoSerializer(TestRecord.class, new TestRecordSerializer());
	executionConfig.registerKryoType(TestRecord.class);

	KryoSerializer<TestRecord> kryoSerializer = new KryoSerializer<TestRecord>(
		TestRecord.class,
		executionConfig);

	int size = 94;
	int bufferSize = 150;

	TestRecord testRecord = new TestRecord(size);

	TestDataOutputView target = new TestDataOutputView(bufferSize);

	kryoSerializer.serialize(testRecord, target);

	try {
		kryoSerializer.serialize(testRecord, target);
		Assert.fail("Expected an EOFException.");
	} catch(EOFException eofException) {
		// expected exception
		// now the Kryo Output should have been cleared
	}

	TestRecord actualRecord = kryoSerializer.deserialize(
			new DataInputViewStreamWrapper(new ByteArrayInputStream(target.getBuffer())));

	Assert.assertEquals(testRecord, actualRecord);

	target.clear();

	// if the kryo output has been cleared then we can serialize our test record into the target
	// because the target buffer 150 bytes can host one TestRecord (total serialization size 100)
	kryoSerializer.serialize(testRecord, target);

	byte[] buffer = target.getBuffer();
	int counter = 0;

	for (int i = 0; i < buffer.length; i++) {
		if(buffer[i] == 42) {
			counter++;
		}
	}

	Assert.assertEquals(size, counter);
}