org.apache.flink.runtime.state.RegisteredPriorityQueueStateBackendMetaInfo Java Examples

The following examples show how to use org.apache.flink.runtime.state.RegisteredPriorityQueueStateBackendMetaInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up
private <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> void createInternal(
	RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo) {

	final String stateName = metaInfo.getName();
	final HeapPriorityQueueSet<T> priorityQueue = priorityQueueSetFactory.create(
		stateName,
		metaInfo.getElementSerializer());

	HeapPriorityQueueSnapshotRestoreWrapper<T> wrapper =
		new HeapPriorityQueueSnapshotRestoreWrapper<>(
			priorityQueue,
			metaInfo,
			KeyExtractorFunction.forKeyedObjects(),
			keyGroupRange,
			numberOfKeyGroups);

	registeredPQStates.put(stateName, wrapper);
}
 
Example #2
Source File: HeapKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T> createInternal(
	RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo) {

	final String stateName = metaInfo.getName();
	final HeapPriorityQueueSet<T> priorityQueue = priorityQueueSetFactory.create(
		stateName,
		metaInfo.getElementSerializer());

	HeapPriorityQueueSnapshotRestoreWrapper<T> wrapper =
		new HeapPriorityQueueSnapshotRestoreWrapper<>(
			priorityQueue,
			metaInfo,
			KeyExtractorFunction.forKeyedObjects(),
			keyGroupRange,
			numberOfKeyGroups);

	registeredPQStates.put(stateName, wrapper);
	return priorityQueue;
}
 
Example #3
Source File: HeapKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Nonnull
private <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T> createInternal(
	RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo) {

	final String stateName = metaInfo.getName();
	final HeapPriorityQueueSet<T> priorityQueue = priorityQueueSetFactory.create(
		stateName,
		metaInfo.getElementSerializer());

	HeapPriorityQueueSnapshotRestoreWrapper<T> wrapper =
		new HeapPriorityQueueSnapshotRestoreWrapper<>(
			priorityQueue,
			metaInfo,
			KeyExtractorFunction.forKeyedObjects(),
			keyGroupRange,
			numberOfKeyGroups);

	registeredPQStates.put(stateName, wrapper);
	return priorityQueue;
}
 
Example #4
Source File: HeapRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> void createInternal(
	RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo) {

	final String stateName = metaInfo.getName();
	final HeapPriorityQueueSet<T> priorityQueue = priorityQueueSetFactory.create(
		stateName,
		metaInfo.getElementSerializer());

	HeapPriorityQueueSnapshotRestoreWrapper<T> wrapper =
		new HeapPriorityQueueSnapshotRestoreWrapper<>(
			priorityQueue,
			metaInfo,
			KeyExtractorFunction.forKeyedObjects(),
			keyGroupRange,
			numberOfKeyGroups);

	registeredPQStates.put(stateName, wrapper);
}
 
Example #5
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up
private <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> void createInternal(
	RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo) {

	final String stateName = metaInfo.getName();
	final HeapPriorityQueueSet<T> priorityQueue = priorityQueueSetFactory.create(
		stateName,
		metaInfo.getElementSerializer());

	HeapPriorityQueueSnapshotRestoreWrapper<T> wrapper =
		new HeapPriorityQueueSnapshotRestoreWrapper<>(
			priorityQueue,
			metaInfo,
			KeyExtractorFunction.forKeyedObjects(),
			keyGroupRange,
			numberOfKeyGroups);

	registeredPQStates.put(stateName, wrapper);
}
 
Example #6
Source File: HeapKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T> createInternal(
	RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo) {

	final String stateName = metaInfo.getName();
	final HeapPriorityQueueSet<T> priorityQueue = priorityQueueSetFactory.create(
		stateName,
		metaInfo.getElementSerializer());

	HeapPriorityQueueSnapshotRestoreWrapper<T> wrapper =
		new HeapPriorityQueueSnapshotRestoreWrapper<>(
			priorityQueue,
			metaInfo,
			KeyExtractorFunction.forKeyedObjects(),
			keyGroupRange,
			numberOfKeyGroups);

	registeredPQStates.put(stateName, wrapper);
	return priorityQueue;
}
 
Example #7
Source File: HeapKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Nonnull
@Override
public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T> create(
	@Nonnull String stateName,
	@Nonnull TypeSerializer<T> byteOrderedElementSerializer) {

	final HeapPriorityQueueSnapshotRestoreWrapper existingState = registeredPQStates.get(stateName);

	if (existingState != null) {
		// TODO we implement the simple way of supporting the current functionality, mimicking keyed state
		// because this should be reworked in FLINK-9376 and then we should have a common algorithm over
		// StateMetaInfoSnapshot that avoids this code duplication.

		TypeSerializerSchemaCompatibility<T> compatibilityResult =
			existingState.getMetaInfo().updateElementSerializer(byteOrderedElementSerializer);

		if (compatibilityResult.isIncompatible()) {
			throw new FlinkRuntimeException(new StateMigrationException("For heap backends, the new priority queue serializer must not be incompatible."));
		} else {
			registeredPQStates.put(
				stateName,
				existingState.forUpdatedSerializer(byteOrderedElementSerializer));
		}

		return existingState.getPriorityQueue();
	} else {
		final RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo =
			new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer);
		return createInternal(metaInfo);
	}
}
 
Example #8
Source File: HeapPriorityQueueStateSnapshot.java    From flink with Apache License 2.0 5 votes vote down vote up
HeapPriorityQueueStateSnapshot(
	@Nonnull T[] heapArrayCopy,
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnull RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups) {

	this.keyExtractor = keyExtractor;
	this.heapArrayCopy = heapArrayCopy;
	this.metaInfo = metaInfo;
	this.keyGroupRange = keyGroupRange;
	this.totalKeyGroups = totalKeyGroups;
}
 
Example #9
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
private void createOrCheckStateForMetaInfo(
	List<StateMetaInfoSnapshot> restoredMetaInfo,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById) {

	for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
		final StateSnapshotRestore registeredState;

		switch (metaInfoSnapshot.getBackendStateType()) {
			case KEY_VALUE:
				registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo =
						new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
					registeredKVStates.put(
						metaInfoSnapshot.getName(),
						snapshotStrategy.newStateTable(
							keyContext,
							registeredKeyedBackendStateMetaInfo,
							keySerializerProvider.currentSchemaSerializer()));
				}
				break;
			case PRIORITY_QUEUE:
				registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot));
				}
				break;
			default:
				throw new IllegalStateException("Unexpected state type: " +
					metaInfoSnapshot.getBackendStateType() + ".");
		}

		// always put metaInfo into kvStatesById, because kvStatesById is KeyGroupsStateHandle related
		kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
	}
}
 
Example #10
Source File: HeapKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Nonnull
@Override
public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T> create(
	@Nonnull String stateName,
	@Nonnull TypeSerializer<T> byteOrderedElementSerializer) {

	final HeapPriorityQueueSnapshotRestoreWrapper existingState = registeredPQStates.get(stateName);

	if (existingState != null) {
		// TODO we implement the simple way of supporting the current functionality, mimicking keyed state
		// because this should be reworked in FLINK-9376 and then we should have a common algorithm over
		// StateMetaInfoSnapshot that avoids this code duplication.

		TypeSerializerSchemaCompatibility<T> compatibilityResult =
			existingState.getMetaInfo().updateElementSerializer(byteOrderedElementSerializer);

		if (compatibilityResult.isIncompatible()) {
			throw new FlinkRuntimeException(new StateMigrationException("For heap backends, the new priority queue serializer must not be incompatible."));
		} else {
			registeredPQStates.put(
				stateName,
				existingState.forUpdatedSerializer(byteOrderedElementSerializer));
		}

		return existingState.getPriorityQueue();
	} else {
		final RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo =
			new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer);
		return createInternal(metaInfo);
	}
}
 
Example #11
Source File: HeapPriorityQueueSnapshotRestoreWrapper.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a deep copy of the snapshot, where the serializer is changed to the given serializer.
 */
public HeapPriorityQueueSnapshotRestoreWrapper<T> forUpdatedSerializer(
	@Nonnull TypeSerializer<T> updatedSerializer) {

	RegisteredPriorityQueueStateBackendMetaInfo<T> updatedMetaInfo =
		new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfo.getName(), updatedSerializer);

	return new HeapPriorityQueueSnapshotRestoreWrapper<>(
		priorityQueue,
		updatedMetaInfo,
		keyExtractorFunction,
		localKeyGroupRange,
		totalKeyGroups);
}
 
Example #12
Source File: HeapPriorityQueueSnapshotRestoreWrapper.java    From flink with Apache License 2.0 5 votes vote down vote up
public HeapPriorityQueueSnapshotRestoreWrapper(
	@Nonnull HeapPriorityQueueSet<T> priorityQueue,
	@Nonnull RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo,
	@Nonnull KeyExtractorFunction<T> keyExtractorFunction,
	@Nonnull KeyGroupRange localKeyGroupRange,
	int totalKeyGroups) {

	this.priorityQueue = priorityQueue;
	this.keyExtractorFunction = keyExtractorFunction;
	this.metaInfo = metaInfo;
	this.localKeyGroupRange = localKeyGroupRange;
	this.totalKeyGroups = totalKeyGroups;
}
 
Example #13
Source File: HeapPriorityQueueStateSnapshot.java    From flink with Apache License 2.0 5 votes vote down vote up
HeapPriorityQueueStateSnapshot(
	@Nonnull T[] heapArrayCopy,
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnull RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups) {

	this.keyExtractor = keyExtractor;
	this.heapArrayCopy = heapArrayCopy;
	this.metaInfo = metaInfo;
	this.keyGroupRange = keyGroupRange;
	this.totalKeyGroups = totalKeyGroups;
}
 
Example #14
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
private void createOrCheckStateForMetaInfo(
	List<StateMetaInfoSnapshot> restoredMetaInfo,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById) {

	for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
		final StateSnapshotRestore registeredState;

		switch (metaInfoSnapshot.getBackendStateType()) {
			case KEY_VALUE:
				registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo =
						new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
					registeredKVStates.put(
						metaInfoSnapshot.getName(),
						snapshotStrategy.newStateTable(
							keyContext,
							registeredKeyedBackendStateMetaInfo,
							keySerializerProvider.currentSchemaSerializer()));
				}
				break;
			case PRIORITY_QUEUE:
				registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot));
				}
				break;
			default:
				throw new IllegalStateException("Unexpected state type: " +
					metaInfoSnapshot.getBackendStateType() + ".");
		}

		if (registeredState == null) {
			kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
		}
	}
}
 
Example #15
Source File: HeapPriorityQueueSnapshotRestoreWrapper.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a deep copy of the snapshot, where the serializer is changed to the given serializer.
 */
public HeapPriorityQueueSnapshotRestoreWrapper<T> forUpdatedSerializer(
	@Nonnull TypeSerializer<T> updatedSerializer) {

	RegisteredPriorityQueueStateBackendMetaInfo<T> updatedMetaInfo =
		new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfo.getName(), updatedSerializer);

	return new HeapPriorityQueueSnapshotRestoreWrapper<>(
		priorityQueue,
		updatedMetaInfo,
		keyExtractorFunction,
		localKeyGroupRange,
		totalKeyGroups);
}
 
Example #16
Source File: HeapPriorityQueueSnapshotRestoreWrapper.java    From flink with Apache License 2.0 5 votes vote down vote up
public HeapPriorityQueueSnapshotRestoreWrapper(
	@Nonnull HeapPriorityQueueSet<T> priorityQueue,
	@Nonnull RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo,
	@Nonnull KeyExtractorFunction<T> keyExtractorFunction,
	@Nonnull KeyGroupRange localKeyGroupRange,
	int totalKeyGroups) {

	this.priorityQueue = priorityQueue;
	this.keyExtractorFunction = keyExtractorFunction;
	this.metaInfo = metaInfo;
	this.localKeyGroupRange = localKeyGroupRange;
	this.totalKeyGroups = totalKeyGroups;
}
 
Example #17
Source File: HeapPriorityQueueStateSnapshot.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
HeapPriorityQueueStateSnapshot(
	@Nonnull T[] heapArrayCopy,
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnull RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups) {

	this.keyExtractor = keyExtractor;
	this.heapArrayCopy = heapArrayCopy;
	this.metaInfo = metaInfo;
	this.keyGroupRange = keyGroupRange;
	this.totalKeyGroups = totalKeyGroups;
}
 
Example #18
Source File: HeapRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void createOrCheckStateForMetaInfo(
	List<StateMetaInfoSnapshot> restoredMetaInfo,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById) {

	for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
		final StateSnapshotRestore registeredState;

		switch (metaInfoSnapshot.getBackendStateType()) {
			case KEY_VALUE:
				registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo =
						new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
					registeredKVStates.put(
						metaInfoSnapshot.getName(),
						snapshotStrategy.newStateTable(backend, registeredKeyedBackendStateMetaInfo));
				}
				break;
			case PRIORITY_QUEUE:
				registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot));
				}
				break;
			default:
				throw new IllegalStateException("Unexpected state type: " +
					metaInfoSnapshot.getBackendStateType() + ".");
		}

		if (registeredState == null) {
			kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
		}
	}
}
 
Example #19
Source File: HeapKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Nonnull
@Override
public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T> create(
	@Nonnull String stateName,
	@Nonnull TypeSerializer<T> byteOrderedElementSerializer) {

	final HeapPriorityQueueSnapshotRestoreWrapper existingState = registeredPQStates.get(stateName);

	if (existingState != null) {
		// TODO we implement the simple way of supporting the current functionality, mimicking keyed state
		// because this should be reworked in FLINK-9376 and then we should have a common algorithm over
		// StateMetaInfoSnapshot that avoids this code duplication.

		TypeSerializerSchemaCompatibility<T> compatibilityResult =
			existingState.getMetaInfo().updateElementSerializer(byteOrderedElementSerializer);

		if (compatibilityResult.isIncompatible()) {
			throw new FlinkRuntimeException(new StateMigrationException("For heap backends, the new priority queue serializer must not be incompatible."));
		} else {
			registeredPQStates.put(
				stateName,
				existingState.forUpdatedSerializer(byteOrderedElementSerializer));
		}

		return existingState.getPriorityQueue();
	} else {
		final RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo =
			new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer);
		return createInternal(metaInfo);
	}
}
 
Example #20
Source File: HeapPriorityQueueSnapshotRestoreWrapper.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a deep copy of the snapshot, where the serializer is changed to the given serializer.
 */
public HeapPriorityQueueSnapshotRestoreWrapper<T> forUpdatedSerializer(
	@Nonnull TypeSerializer<T> updatedSerializer) {

	RegisteredPriorityQueueStateBackendMetaInfo<T> updatedMetaInfo =
		new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfo.getName(), updatedSerializer);

	return new HeapPriorityQueueSnapshotRestoreWrapper<>(
		priorityQueue,
		updatedMetaInfo,
		keyExtractorFunction,
		localKeyGroupRange,
		totalKeyGroups);
}
 
Example #21
Source File: HeapPriorityQueueSnapshotRestoreWrapper.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public HeapPriorityQueueSnapshotRestoreWrapper(
	@Nonnull HeapPriorityQueueSet<T> priorityQueue,
	@Nonnull RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo,
	@Nonnull KeyExtractorFunction<T> keyExtractorFunction,
	@Nonnull KeyGroupRange localKeyGroupRange,
	int totalKeyGroups) {

	this.priorityQueue = priorityQueue;
	this.keyExtractorFunction = keyExtractorFunction;
	this.metaInfo = metaInfo;
	this.localKeyGroupRange = localKeyGroupRange;
	this.totalKeyGroups = totalKeyGroups;
}
 
Example #22
Source File: RocksDBPriorityQueueSetFactory.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Nonnull
private <T> RocksDBKeyedStateBackend.RocksDbKvStateInfo tryRegisterPriorityQueueMetaInfo(
	@Nonnull String stateName,
	@Nonnull TypeSerializer<T> byteOrderedElementSerializer) {

	RocksDBKeyedStateBackend.RocksDbKvStateInfo stateInfo = kvStateInformation.get(stateName);

	if (stateInfo == null) {
		// Currently this class is for timer service and TTL feature is not applicable here,
		// so no need to register compact filter when creating column family
		RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo =
			new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer);
		stateInfo = RocksDBOperationUtils.createStateInfo(metaInfo, db, columnFamilyOptionsFactory, null);
		RocksDBOperationUtils.registerKvStateInformation(kvStateInformation, nativeMetricMonitor, stateName, stateInfo);
	} else {
		// TODO we implement the simple way of supporting the current functionality, mimicking keyed state
		// because this should be reworked in FLINK-9376 and then we should have a common algorithm over
		// StateMetaInfoSnapshot that avoids this code duplication.

		@SuppressWarnings("unchecked")
		RegisteredPriorityQueueStateBackendMetaInfo<T> castedMetaInfo =
			(RegisteredPriorityQueueStateBackendMetaInfo<T>) stateInfo.metaInfo;

		TypeSerializer<T> previousElementSerializer = castedMetaInfo.getPreviousElementSerializer();

		if (previousElementSerializer != byteOrderedElementSerializer) {
			TypeSerializerSchemaCompatibility<T> compatibilityResult =
				castedMetaInfo.updateElementSerializer(byteOrderedElementSerializer);

			// Since priority queue elements are written into RocksDB
			// as keys prefixed with the key group and namespace, we do not support
			// migrating them. Therefore, here we only check for incompatibility.
			if (compatibilityResult.isIncompatible()) {
				throw new FlinkRuntimeException(
					new StateMigrationException("The new priority queue serializer must not be incompatible."));
			}

			// update meta info with new serializer
			stateInfo = new RocksDBKeyedStateBackend.RocksDbKvStateInfo(
				stateInfo.columnFamilyHandle,
				new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer));
			kvStateInformation.put(stateName, stateInfo);
		}
	}

	return stateInfo;
}
 
Example #23
Source File: HeapPriorityQueueSnapshotRestoreWrapper.java    From flink with Apache License 2.0 4 votes vote down vote up
@Nonnull
public RegisteredPriorityQueueStateBackendMetaInfo<T> getMetaInfo() {
	return metaInfo;
}
 
Example #24
Source File: RocksDBPriorityQueueSetFactory.java    From flink with Apache License 2.0 4 votes vote down vote up
@Nonnull
private <T> RocksDBKeyedStateBackend.RocksDbKvStateInfo tryRegisterPriorityQueueMetaInfo(
	@Nonnull String stateName,
	@Nonnull TypeSerializer<T> byteOrderedElementSerializer) {

	RocksDBKeyedStateBackend.RocksDbKvStateInfo stateInfo = kvStateInformation.get(stateName);

	if (stateInfo == null) {
		// Currently this class is for timer service and TTL feature is not applicable here,
		// so no need to register compact filter when creating column family
		RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo =
			new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer);
		stateInfo = RocksDBOperationUtils.createStateInfo(metaInfo, db, columnFamilyOptionsFactory, null);
		RocksDBOperationUtils.registerKvStateInformation(kvStateInformation, nativeMetricMonitor, stateName, stateInfo);
	} else {
		// TODO we implement the simple way of supporting the current functionality, mimicking keyed state
		// because this should be reworked in FLINK-9376 and then we should have a common algorithm over
		// StateMetaInfoSnapshot that avoids this code duplication.

		@SuppressWarnings("unchecked")
		RegisteredPriorityQueueStateBackendMetaInfo<T> castedMetaInfo =
			(RegisteredPriorityQueueStateBackendMetaInfo<T>) stateInfo.metaInfo;

		TypeSerializer<T> previousElementSerializer = castedMetaInfo.getPreviousElementSerializer();

		if (previousElementSerializer != byteOrderedElementSerializer) {
			TypeSerializerSchemaCompatibility<T> compatibilityResult =
				castedMetaInfo.updateElementSerializer(byteOrderedElementSerializer);

			// Since priority queue elements are written into RocksDB
			// as keys prefixed with the key group and namespace, we do not support
			// migrating them. Therefore, here we only check for incompatibility.
			if (compatibilityResult.isIncompatible()) {
				throw new FlinkRuntimeException(
					new StateMigrationException("The new priority queue serializer must not be incompatible."));
			}

			// update meta info with new serializer
			stateInfo = new RocksDBKeyedStateBackend.RocksDbKvStateInfo(
				stateInfo.columnFamilyHandle,
				new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer));
			kvStateInformation.put(stateName, stateInfo);
		}
	}

	return stateInfo;
}
 
Example #25
Source File: RocksDBPriorityQueueSetFactory.java    From flink with Apache License 2.0 4 votes vote down vote up
@Nonnull
private <T> RocksDBKeyedStateBackend.RocksDbKvStateInfo tryRegisterPriorityQueueMetaInfo(
	@Nonnull String stateName,
	@Nonnull TypeSerializer<T> byteOrderedElementSerializer) {

	RocksDBKeyedStateBackend.RocksDbKvStateInfo stateInfo = kvStateInformation.get(stateName);

	if (stateInfo == null) {
		// Currently this class is for timer service and TTL feature is not applicable here,
		// so no need to register compact filter when creating column family
		RegisteredPriorityQueueStateBackendMetaInfo<T> metaInfo =
			new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer);
		stateInfo = RocksDBOperationUtils.createStateInfo(metaInfo, db, columnFamilyOptionsFactory, null);
		RocksDBOperationUtils.registerKvStateInformation(kvStateInformation, nativeMetricMonitor, stateName, stateInfo);
	} else {
		// TODO we implement the simple way of supporting the current functionality, mimicking keyed state
		// because this should be reworked in FLINK-9376 and then we should have a common algorithm over
		// StateMetaInfoSnapshot that avoids this code duplication.

		@SuppressWarnings("unchecked")
		RegisteredPriorityQueueStateBackendMetaInfo<T> castedMetaInfo =
			(RegisteredPriorityQueueStateBackendMetaInfo<T>) stateInfo.metaInfo;

		TypeSerializer<T> previousElementSerializer = castedMetaInfo.getPreviousElementSerializer();

		if (previousElementSerializer != byteOrderedElementSerializer) {
			TypeSerializerSchemaCompatibility<T> compatibilityResult =
				castedMetaInfo.updateElementSerializer(byteOrderedElementSerializer);

			// Since priority queue elements are written into RocksDB
			// as keys prefixed with the key group and namespace, we do not support
			// migrating them. Therefore, here we only check for incompatibility.
			if (compatibilityResult.isIncompatible()) {
				throw new FlinkRuntimeException(
					new StateMigrationException("The new priority queue serializer must not be incompatible."));
			}

			// update meta info with new serializer
			stateInfo = new RocksDBKeyedStateBackend.RocksDbKvStateInfo(
				stateInfo.columnFamilyHandle,
				new RegisteredPriorityQueueStateBackendMetaInfo<>(stateName, byteOrderedElementSerializer));
			kvStateInformation.put(stateName, stateInfo);
		}
	}

	return stateInfo;
}
 
Example #26
Source File: HeapPriorityQueueSnapshotRestoreWrapper.java    From flink with Apache License 2.0 4 votes vote down vote up
@Nonnull
public RegisteredPriorityQueueStateBackendMetaInfo<T> getMetaInfo() {
	return metaInfo;
}
 
Example #27
Source File: HeapPriorityQueueSnapshotRestoreWrapper.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Nonnull
public RegisteredPriorityQueueStateBackendMetaInfo<T> getMetaInfo() {
	return metaInfo;
}