Java Code Examples for org.apache.flink.api.common.state.StateDescriptor#getSerializer()

@Test
public void testValueStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class);
	context.getState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 

@Test
public void testValueStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class);
	context.getState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 

@Test
public void testValueStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = createRuntimeContext(descriptorCapture, config);
	ValueStateDescriptor<TaskInfo> descr = new ValueStateDescriptor<>("name", TaskInfo.class);
	context.getState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 

private void validateStateDescriptorConfigured(SingleOutputStreamOperator<?> result) {
	OneInputTransformation<?, ?> transform = (OneInputTransformation<?, ?>) result.getTransformation();
	WindowOperator<?, ?, ?, ?, ?> op = (WindowOperator<?, ?, ?, ?, ?>) transform.getOperator();
	StateDescriptor<?, ?> descr = op.getStateDescriptor();

	// this would be the first statement to fail if state descriptors were not properly initialized
	TypeSerializer<?> serializer = descr.getSerializer();
	assertTrue(serializer instanceof KryoSerializer);

	Kryo kryo = ((KryoSerializer<?>) serializer).getKryo();

	assertTrue("serializer registration was not properly passed on",
			kryo.getSerializer(File.class) instanceof JavaSerializer);
}
 

@Test
public void testReducingStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	@SuppressWarnings("unchecked")
	ReduceFunction<TaskInfo> reducer = (ReduceFunction<TaskInfo>) mock(ReduceFunction.class);

	ReducingStateDescriptor<TaskInfo> descr =
			new ReducingStateDescriptor<>("name", reducer, TaskInfo.class);

	context.getReducingState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 

private void validateStateDescriptorConfigured(SingleOutputStreamOperator<?> result) {
	OneInputTransformation<?, ?> transform = (OneInputTransformation<?, ?>) result.getTransformation();
	WindowOperator<?, ?, ?, ?, ?> op = (WindowOperator<?, ?, ?, ?, ?>) transform.getOperator();
	StateDescriptor<?, ?> descr = op.getStateDescriptor();

	// this would be the first statement to fail if state descriptors were not properly initialized
	TypeSerializer<?> serializer = descr.getSerializer();
	assertTrue(serializer instanceof KryoSerializer);

	Kryo kryo = ((KryoSerializer<?>) serializer).getKryo();

	assertTrue("serializer registration was not properly passed on",
			kryo.getSerializer(File.class) instanceof JavaSerializer);
}
 

@Test
public void testReducingStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	@SuppressWarnings("unchecked")
	ReduceFunction<TaskInfo> reducer = (ReduceFunction<TaskInfo>) mock(ReduceFunction.class);

	ReducingStateDescriptor<TaskInfo> descr =
			new ReducingStateDescriptor<>("name", reducer, TaskInfo.class);

	context.getReducingState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 

private void validateStateDescriptorConfigured(SingleOutputStreamOperator<?> result) {
	OneInputTransformation<?, ?> transform = (OneInputTransformation<?, ?>) result.getTransformation();
	WindowOperator<?, ?, ?, ?, ?> op = (WindowOperator<?, ?, ?, ?, ?>) transform.getOperator();
	StateDescriptor<?, ?> descr = op.getStateDescriptor();

	// this would be the first statement to fail if state descriptors were not properly initialized
	TypeSerializer<?> serializer = descr.getSerializer();
	assertTrue(serializer instanceof KryoSerializer);

	Kryo kryo = ((KryoSerializer<?>) serializer).getKryo();

	assertTrue("serializer registration was not properly passed on",
			kryo.getSerializer(File.class) instanceof JavaSerializer);
}
 

/**
 * Registers a k/v state information, which includes its state id, type, RocksDB column family handle, and serializers.
 *
 * <p>When restoring from a snapshot, we don’t restore the individual k/v states, just the global RocksDB database and
 * the list of k/v state information. When a k/v state is first requested we check here whether we
 * already have a registered entry for that and return it (after some necessary state compatibility checks)
 * or create a new one if it does not exist.
 */
private <N, S extends State, SV, SEV> Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> tryRegisterKvStateInformation(
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {

	RocksDbKvStateInfo oldStateInfo = kvStateInformation.get(stateDesc.getName());

	TypeSerializer<SV> stateSerializer = stateDesc.getSerializer();

	RocksDbKvStateInfo newRocksStateInfo;
	RegisteredKeyValueStateBackendMetaInfo<N, SV> newMetaInfo;
	if (oldStateInfo != null) {
		@SuppressWarnings("unchecked")
		RegisteredKeyValueStateBackendMetaInfo<N, SV> castedMetaInfo =
			(RegisteredKeyValueStateBackendMetaInfo<N, SV>) oldStateInfo.metaInfo;

		newMetaInfo = updateRestoredStateMetaInfo(
			Tuple2.of(oldStateInfo.columnFamilyHandle, castedMetaInfo),
			stateDesc,
			namespaceSerializer,
			stateSerializer);

		newRocksStateInfo = new RocksDbKvStateInfo(oldStateInfo.columnFamilyHandle, newMetaInfo);
		kvStateInformation.put(stateDesc.getName(), newRocksStateInfo);
	} else {
		newMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
			stateDesc.getType(),
			stateDesc.getName(),
			namespaceSerializer,
			stateSerializer,
			StateSnapshotTransformFactory.noTransform());

		newRocksStateInfo = RocksDBOperationUtils.createStateInfo(
			newMetaInfo, db, columnFamilyOptionsFactory, ttlCompactFiltersManager);
		RocksDBOperationUtils.registerKvStateInformation(this.kvStateInformation, this.nativeMetricMonitor,
			stateDesc.getName(), newRocksStateInfo);
	}

	StateSnapshotTransformFactory<SV> wrappedSnapshotTransformFactory = wrapStateSnapshotTransformFactory(
		stateDesc, snapshotTransformFactory, newMetaInfo.getStateSerializer());
	newMetaInfo.updateSnapshotTransformFactory(wrappedSnapshotTransformFactory);

	ttlCompactFiltersManager.configCompactFilter(stateDesc, newMetaInfo.getStateSerializer());

	return Tuple2.of(newRocksStateInfo.columnFamilyHandle, newMetaInfo);
}
 

private <N, V> StateTable<K, N, V> tryRegisterStateTable(
	TypeSerializer<N> namespaceSerializer,
	StateDescriptor<?, V> stateDesc,
	@Nonnull StateSnapshotTransformFactory<V> snapshotTransformFactory) throws StateMigrationException {

	@SuppressWarnings("unchecked")
	StateTable<K, N, V> stateTable = (StateTable<K, N, V>) registeredKVStates.get(stateDesc.getName());

	TypeSerializer<V> newStateSerializer = stateDesc.getSerializer();

	if (stateTable != null) {
		RegisteredKeyValueStateBackendMetaInfo<N, V> restoredKvMetaInfo = stateTable.getMetaInfo();

		restoredKvMetaInfo.updateSnapshotTransformFactory(snapshotTransformFactory);

		TypeSerializerSchemaCompatibility<N> namespaceCompatibility =
			restoredKvMetaInfo.updateNamespaceSerializer(namespaceSerializer);
		if (namespaceCompatibility.isCompatibleAfterMigration() || namespaceCompatibility.isIncompatible()) {
			throw new StateMigrationException("For heap backends, the new namespace serializer must be compatible.");
		}

		restoredKvMetaInfo.checkStateMetaInfo(stateDesc);

		TypeSerializerSchemaCompatibility<V> stateCompatibility =
			restoredKvMetaInfo.updateStateSerializer(newStateSerializer);

		if (stateCompatibility.isIncompatible()) {
			throw new StateMigrationException("For heap backends, the new state serializer must not be incompatible.");
		}

		stateTable.setMetaInfo(restoredKvMetaInfo);
	} else {
		RegisteredKeyValueStateBackendMetaInfo<N, V> newMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
			stateDesc.getType(),
			stateDesc.getName(),
			namespaceSerializer,
			newStateSerializer,
			snapshotTransformFactory);

		stateTable = snapshotStrategy.newStateTable(this, newMetaInfo);
		registeredKVStates.put(stateDesc.getName(), stateTable);
	}

	return stateTable;
}
 

/**
 * Registers a k/v state information, which includes its state id, type, RocksDB column family handle, and serializers.
 *
 * <p>When restoring from a snapshot, we don’t restore the individual k/v states, just the global RocksDB database and
 * the list of k/v state information. When a k/v state is first requested we check here whether we
 * already have a registered entry for that and return it (after some necessary state compatibility checks)
 * or create a new one if it does not exist.
 */
private <N, S extends State, SV, SEV> Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> tryRegisterKvStateInformation(
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {

	RocksDbKvStateInfo oldStateInfo = kvStateInformation.get(stateDesc.getName());

	TypeSerializer<SV> stateSerializer = stateDesc.getSerializer();

	RocksDbKvStateInfo newRocksStateInfo;
	RegisteredKeyValueStateBackendMetaInfo<N, SV> newMetaInfo;
	if (oldStateInfo != null) {
		@SuppressWarnings("unchecked")
		RegisteredKeyValueStateBackendMetaInfo<N, SV> castedMetaInfo =
			(RegisteredKeyValueStateBackendMetaInfo<N, SV>) oldStateInfo.metaInfo;

		newMetaInfo = updateRestoredStateMetaInfo(
			Tuple2.of(oldStateInfo.columnFamilyHandle, castedMetaInfo),
			stateDesc,
			namespaceSerializer,
			stateSerializer);

		newRocksStateInfo = new RocksDbKvStateInfo(oldStateInfo.columnFamilyHandle, newMetaInfo);
		kvStateInformation.put(stateDesc.getName(), newRocksStateInfo);
	} else {
		newMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
			stateDesc.getType(),
			stateDesc.getName(),
			namespaceSerializer,
			stateSerializer,
			StateSnapshotTransformFactory.noTransform());

		newRocksStateInfo = RocksDBOperationUtils.createStateInfo(
			newMetaInfo, db, columnFamilyOptionsFactory, ttlCompactFiltersManager);
		RocksDBOperationUtils.registerKvStateInformation(this.kvStateInformation, this.nativeMetricMonitor,
			stateDesc.getName(), newRocksStateInfo);
	}

	StateSnapshotTransformFactory<SV> wrappedSnapshotTransformFactory = wrapStateSnapshotTransformFactory(
		stateDesc, snapshotTransformFactory, newMetaInfo.getStateSerializer());
	newMetaInfo.updateSnapshotTransformFactory(wrappedSnapshotTransformFactory);

	ttlCompactFiltersManager.configCompactFilter(stateDesc, newMetaInfo.getStateSerializer());

	return Tuple2.of(newRocksStateInfo.columnFamilyHandle, newMetaInfo);
}
 

private <N, V> StateTable<K, N, V> tryRegisterStateTable(
	TypeSerializer<N> namespaceSerializer,
	StateDescriptor<?, V> stateDesc,
	@Nonnull StateSnapshotTransformFactory<V> snapshotTransformFactory) throws StateMigrationException {

	@SuppressWarnings("unchecked")
	StateTable<K, N, V> stateTable = (StateTable<K, N, V>) registeredKVStates.get(stateDesc.getName());

	TypeSerializer<V> newStateSerializer = stateDesc.getSerializer();

	if (stateTable != null) {
		RegisteredKeyValueStateBackendMetaInfo<N, V> restoredKvMetaInfo = stateTable.getMetaInfo();

		restoredKvMetaInfo.updateSnapshotTransformFactory(snapshotTransformFactory);

		TypeSerializerSchemaCompatibility<N> namespaceCompatibility =
			restoredKvMetaInfo.updateNamespaceSerializer(namespaceSerializer);
		if (namespaceCompatibility.isCompatibleAfterMigration() || namespaceCompatibility.isIncompatible()) {
			throw new StateMigrationException("For heap backends, the new namespace serializer must be compatible.");
		}

		restoredKvMetaInfo.checkStateMetaInfo(stateDesc);

		TypeSerializerSchemaCompatibility<V> stateCompatibility =
			restoredKvMetaInfo.updateStateSerializer(newStateSerializer);

		if (stateCompatibility.isIncompatible()) {
			throw new StateMigrationException("For heap backends, the new state serializer must not be incompatible.");
		}

		stateTable.setMetaInfo(restoredKvMetaInfo);
	} else {
		RegisteredKeyValueStateBackendMetaInfo<N, V> newMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
			stateDesc.getType(),
			stateDesc.getName(),
			namespaceSerializer,
			newStateSerializer,
			snapshotTransformFactory);

		stateTable = snapshotStrategy.newStateTable(keyContext, newMetaInfo, keySerializer);
		registeredKVStates.put(stateDesc.getName(), stateTable);
	}

	return stateTable;
}
 

/**
 * Registers a k/v state information, which includes its state id, type, RocksDB column family handle, and serializers.
 *
 * <p>When restoring from a snapshot, we don’t restore the individual k/v states, just the global RocksDB database and
 * the list of k/v state information. When a k/v state is first requested we check here whether we
 * already have a registered entry for that and return it (after some necessary state compatibility checks)
 * or create a new one if it does not exist.
 */
private <N, S extends State, SV, SEV> Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> tryRegisterKvStateInformation(
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {

	RocksDbKvStateInfo oldStateInfo = kvStateInformation.get(stateDesc.getName());

	TypeSerializer<SV> stateSerializer = stateDesc.getSerializer();

	RocksDbKvStateInfo newRocksStateInfo;
	RegisteredKeyValueStateBackendMetaInfo<N, SV> newMetaInfo;
	if (oldStateInfo != null) {
		@SuppressWarnings("unchecked")
		RegisteredKeyValueStateBackendMetaInfo<N, SV> castedMetaInfo =
			(RegisteredKeyValueStateBackendMetaInfo<N, SV>) oldStateInfo.metaInfo;

		newMetaInfo = updateRestoredStateMetaInfo(
			Tuple2.of(oldStateInfo.columnFamilyHandle, castedMetaInfo),
			stateDesc,
			namespaceSerializer,
			stateSerializer);

		newRocksStateInfo = new RocksDbKvStateInfo(oldStateInfo.columnFamilyHandle, newMetaInfo);
		kvStateInformation.put(stateDesc.getName(), newRocksStateInfo);
	} else {
		newMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
			stateDesc.getType(),
			stateDesc.getName(),
			namespaceSerializer,
			stateSerializer,
			StateSnapshotTransformFactory.noTransform());

		newRocksStateInfo = RocksDBOperationUtils.createStateInfo(
			newMetaInfo, db, columnFamilyOptionsFactory, ttlCompactFiltersManager);
		RocksDBOperationUtils.registerKvStateInformation(this.kvStateInformation, this.nativeMetricMonitor,
			stateDesc.getName(), newRocksStateInfo);
	}

	StateSnapshotTransformFactory<SV> wrappedSnapshotTransformFactory = wrapStateSnapshotTransformFactory(
		stateDesc, snapshotTransformFactory, newMetaInfo.getStateSerializer());
	newMetaInfo.updateSnapshotTransformFactory(wrappedSnapshotTransformFactory);

	ttlCompactFiltersManager.configCompactFilter(stateDesc, newMetaInfo.getStateSerializer());

	return Tuple2.of(newRocksStateInfo.columnFamilyHandle, newMetaInfo);
}
 

private <N, V> StateTable<K, N, V> tryRegisterStateTable(
	TypeSerializer<N> namespaceSerializer,
	StateDescriptor<?, V> stateDesc,
	@Nonnull StateSnapshotTransformFactory<V> snapshotTransformFactory) throws StateMigrationException {

	@SuppressWarnings("unchecked")
	StateTable<K, N, V> stateTable = (StateTable<K, N, V>) registeredKVStates.get(stateDesc.getName());

	TypeSerializer<V> newStateSerializer = stateDesc.getSerializer();

	if (stateTable != null) {
		RegisteredKeyValueStateBackendMetaInfo<N, V> restoredKvMetaInfo = stateTable.getMetaInfo();

		restoredKvMetaInfo.updateSnapshotTransformFactory(snapshotTransformFactory);

		TypeSerializerSchemaCompatibility<N> namespaceCompatibility =
			restoredKvMetaInfo.updateNamespaceSerializer(namespaceSerializer);
		if (namespaceCompatibility.isCompatibleAfterMigration() || namespaceCompatibility.isIncompatible()) {
			throw new StateMigrationException("For heap backends, the new namespace serializer must be compatible.");
		}

		restoredKvMetaInfo.checkStateMetaInfo(stateDesc);

		TypeSerializerSchemaCompatibility<V> stateCompatibility =
			restoredKvMetaInfo.updateStateSerializer(newStateSerializer);

		if (stateCompatibility.isIncompatible()) {
			throw new StateMigrationException("For heap backends, the new state serializer must not be incompatible.");
		}

		stateTable.setMetaInfo(restoredKvMetaInfo);
	} else {
		RegisteredKeyValueStateBackendMetaInfo<N, V> newMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
			stateDesc.getType(),
			stateDesc.getName(),
			namespaceSerializer,
			newStateSerializer,
			snapshotTransformFactory);

		stateTable = snapshotStrategy.newStateTable(keyContext, newMetaInfo, keySerializer);
		registeredKVStates.put(stateDesc.getName(), stateTable);
	}

	return stateTable;
}
 

@Test
public void testReducingStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = createRuntimeContext(descriptorCapture, config);

	@SuppressWarnings("unchecked")
	ReduceFunction<TaskInfo> reducer = (ReduceFunction<TaskInfo>) mock(ReduceFunction.class);

	ReducingStateDescriptor<TaskInfo> descr =
			new ReducingStateDescriptor<>("name", reducer, TaskInfo.class);

	context.getReducingState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}