org.apache.flink.runtime.state.StateSnapshot Java Examples

The following examples show how to use org.apache.flink.runtime.state.StateSnapshot. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HeapSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private void processSnapshotMetaInfoForAllStates(
	List<StateMetaInfoSnapshot> metaInfoSnapshots,
	Map<StateUID, StateSnapshot> cowStateStableSnapshots,
	Map<StateUID, Integer> stateNamesToId,
	Map<String, ? extends StateSnapshotRestore> registeredStates,
	StateMetaInfoSnapshot.BackendStateType stateType) {

	for (Map.Entry<String, ? extends StateSnapshotRestore> kvState : registeredStates.entrySet()) {
		final StateUID stateUid = StateUID.of(kvState.getKey(), stateType);
		stateNamesToId.put(stateUid, stateNamesToId.size());
		StateSnapshotRestore state = kvState.getValue();
		if (null != state) {
			final StateSnapshot stateSnapshot = state.stateSnapshot();
			metaInfoSnapshots.add(stateSnapshot.getMetaInfoSnapshot());
			cowStateStableSnapshots.put(stateUid, stateSnapshot);
		}
	}
}
Example #2
Source File: HeapSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void processSnapshotMetaInfoForAllStates(
	List<StateMetaInfoSnapshot> metaInfoSnapshots,
	Map<StateUID, StateSnapshot> cowStateStableSnapshots,
	Map<StateUID, Integer> stateNamesToId,
	Map<String, ? extends StateSnapshotRestore> registeredStates,
	StateMetaInfoSnapshot.BackendStateType stateType) {

	for (Map.Entry<String, ? extends StateSnapshotRestore> kvState : registeredStates.entrySet()) {
		final StateUID stateUid = StateUID.of(kvState.getKey(), stateType);
		stateNamesToId.put(stateUid, stateNamesToId.size());
		StateSnapshotRestore state = kvState.getValue();
		if (null != state) {
			final StateSnapshot stateSnapshot = state.stateSnapshot();
			metaInfoSnapshots.add(stateSnapshot.getMetaInfoSnapshot());
			cowStateStableSnapshots.put(stateUid, stateSnapshot);
		}
	}
}
Example #3
Source File: HeapSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void processSnapshotMetaInfoForAllStates(
	List<StateMetaInfoSnapshot> metaInfoSnapshots,
	Map<StateUID, StateSnapshot> cowStateStableSnapshots,
	Map<StateUID, Integer> stateNamesToId,
	Map<String, ? extends StateSnapshotRestore> registeredStates,
	StateMetaInfoSnapshot.BackendStateType stateType) {

	for (Map.Entry<String, ? extends StateSnapshotRestore> kvState : registeredStates.entrySet()) {
		final StateUID stateUid = StateUID.of(kvState.getKey(), stateType);
		stateNamesToId.put(stateUid, stateNamesToId.size());
		StateSnapshotRestore state = kvState.getValue();
		if (null != state) {
			final StateSnapshot stateSnapshot = state.stateSnapshot();
			metaInfoSnapshots.add(stateSnapshot.getMetaInfoSnapshot());
			cowStateStableSnapshots.put(stateUid, stateSnapshot);
		}
	}
}
Example #4
Source File: HeapPriorityQueueSnapshotRestoreWrapper.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Nonnull
@Override
public StateSnapshot stateSnapshot() {
	final T[] queueDump = (T[]) priorityQueue.toArray(new HeapPriorityQueueElement[priorityQueue.size()]);
	return new HeapPriorityQueueStateSnapshot<>(
		queueDump,
		keyExtractorFunction,
		metaInfo.deepCopy(),
		localKeyGroupRange,
		totalKeyGroups);
}
Example #5
Source File: HeapPriorityQueueSnapshotRestoreWrapper.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Nonnull
@Override
public StateSnapshot stateSnapshot() {
	final T[] queueDump = (T[]) priorityQueue.toArray(new HeapPriorityQueueElement[priorityQueue.size()]);
	return new HeapPriorityQueueStateSnapshot<>(
		queueDump,
		keyExtractorFunction,
		metaInfo.deepCopy(),
		localKeyGroupRange,
		totalKeyGroups);
}
Example #6
Source File: CopyOnWriteStateTableTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * This tests that serializers used for snapshots are duplicates of the ones used in
 * processing to avoid race conditions in stateful serializers.
 */
@Test
public void testSerializerDuplicationInSnapshot() throws IOException {

	final TestDuplicateSerializer namespaceSerializer = new TestDuplicateSerializer();
	final TestDuplicateSerializer stateSerializer = new TestDuplicateSerializer();
	final TestDuplicateSerializer keySerializer = new TestDuplicateSerializer();

	RegisteredKeyValueStateBackendMetaInfo<Integer, Integer> metaInfo =
		new RegisteredKeyValueStateBackendMetaInfo<>(
			StateDescriptor.Type.VALUE,
			"test",
			namespaceSerializer,
			stateSerializer);

	InternalKeyContext<Integer> mockKeyContext = new MockInternalKeyContext<>();
	CopyOnWriteStateTable<Integer, Integer, Integer> table =
		new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, keySerializer);

	table.put(0, 0, 0, 0);
	table.put(1, 0, 0, 1);
	table.put(2, 0, 1, 2);


	final CopyOnWriteStateTableSnapshot<Integer, Integer, Integer> snapshot = table.stateSnapshot();

	final StateSnapshot.StateKeyGroupWriter partitionedSnapshot = snapshot.getKeyGroupWriter();
	namespaceSerializer.disable();
	keySerializer.disable();
	stateSerializer.disable();

	partitionedSnapshot.writeStateInKeyGroup(
		new DataOutputViewStreamWrapper(
			new ByteArrayOutputStreamWithPos(1024)), 0);
}
Example #7
Source File: HeapPriorityQueueSnapshotRestoreWrapper.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Nonnull
@Override
public StateSnapshot stateSnapshot() {
	final T[] queueDump = (T[]) priorityQueue.toArray(new HeapPriorityQueueElement[priorityQueue.size()]);
	return new HeapPriorityQueueStateSnapshot<>(
		queueDump,
		keyExtractorFunction,
		metaInfo.deepCopy(),
		localKeyGroupRange,
		totalKeyGroups);
}
Example #8
Source File: CopyOnWriteStateTableTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * This tests that serializers used for snapshots are duplicates of the ones used in
 * processing to avoid race conditions in stateful serializers.
 */
@Test
public void testSerializerDuplicationInSnapshot() throws IOException {

	final TestDuplicateSerializer namespaceSerializer = new TestDuplicateSerializer();
	final TestDuplicateSerializer stateSerializer = new TestDuplicateSerializer();
	final TestDuplicateSerializer keySerializer = new TestDuplicateSerializer();

	RegisteredKeyValueStateBackendMetaInfo<Integer, Integer> metaInfo =
		new RegisteredKeyValueStateBackendMetaInfo<>(
			StateDescriptor.Type.VALUE,
			"test",
			namespaceSerializer,
			stateSerializer);

	InternalKeyContext<Integer> mockKeyContext = new MockInternalKeyContext<>();
	CopyOnWriteStateTable<Integer, Integer, Integer> table =
		new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, keySerializer);

	table.put(0, 0, 0, 0);
	table.put(1, 0, 0, 1);
	table.put(2, 0, 1, 2);


	final CopyOnWriteStateTableSnapshot<Integer, Integer, Integer> snapshot = table.stateSnapshot();

	final StateSnapshot.StateKeyGroupWriter partitionedSnapshot = snapshot.getKeyGroupWriter();
	namespaceSerializer.disable();
	keySerializer.disable();
	stateSerializer.disable();

	partitionedSnapshot.writeStateInKeyGroup(
		new DataOutputViewStreamWrapper(
			new ByteArrayOutputStreamWithPos(1024)), 0);
}
Example #9
Source File: CopyOnWriteStateTableTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * This tests that serializers used for snapshots are duplicates of the ones used in
 * processing to avoid race conditions in stateful serializers.
 */
@Test
public void testSerializerDuplicationInSnapshot() throws IOException {

	final TestDuplicateSerializer namespaceSerializer = new TestDuplicateSerializer();
	final TestDuplicateSerializer stateSerializer = new TestDuplicateSerializer();
	final TestDuplicateSerializer keySerializer = new TestDuplicateSerializer();

	RegisteredKeyValueStateBackendMetaInfo<Integer, Integer> metaInfo =
		new RegisteredKeyValueStateBackendMetaInfo<>(
			StateDescriptor.Type.VALUE,
			"test",
			namespaceSerializer,
			stateSerializer);

	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	InternalKeyContext<Integer> mockKeyContext = new InternalKeyContext<Integer>() {
		@Override
		public Integer getCurrentKey() {
			return 0;
		}

		@Override
		public int getCurrentKeyGroupIndex() {
			return 0;
		}

		@Override
		public int getNumberOfKeyGroups() {
			return 1;
		}

		@Override
		public KeyGroupRange getKeyGroupRange() {
			return keyGroupRange;
		}

		@Override
		public TypeSerializer<Integer> getKeySerializer() {
			return keySerializer;
		}
	};

	CopyOnWriteStateTable<Integer, Integer, Integer> table =
		new CopyOnWriteStateTable<>(mockKeyContext, metaInfo);

	table.put(0, 0, 0, 0);
	table.put(1, 0, 0, 1);
	table.put(2, 0, 1, 2);


	final CopyOnWriteStateTableSnapshot<Integer, Integer, Integer> snapshot = table.stateSnapshot();

	try {
		final StateSnapshot.StateKeyGroupWriter partitionedSnapshot = snapshot.getKeyGroupWriter();
		namespaceSerializer.disable();
		keySerializer.disable();
		stateSerializer.disable();

		partitionedSnapshot.writeStateInKeyGroup(
			new DataOutputViewStreamWrapper(
				new ByteArrayOutputStreamWithPos(1024)), 0);

	} finally {
		table.releaseSnapshot(snapshot);
	}
}