Java Code Examples for org.apache.flink.runtime.state.KeyGroupsStateHandle#openInputStream()

The following examples show how to use org.apache.flink.runtime.state.KeyGroupsStateHandle#openInputStream() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0

5 votes

public static void compareKeyedState(
		Collection<KeyGroupsStateHandle> expectPartitionedKeyGroupState,
		Collection<? extends KeyedStateHandle> actualPartitionedKeyGroupState) throws Exception {

	KeyGroupsStateHandle expectedHeadOpKeyGroupStateHandle = expectPartitionedKeyGroupState.iterator().next();
	int expectedTotalKeyGroups = expectedHeadOpKeyGroupStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	int actualTotalKeyGroups = 0;
	for(KeyedStateHandle keyedStateHandle: actualPartitionedKeyGroupState) {
		assertTrue(keyedStateHandle instanceof KeyGroupsStateHandle);

		actualTotalKeyGroups += keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	}

	assertEquals(expectedTotalKeyGroups, actualTotalKeyGroups);

	try (FSDataInputStream inputStream = expectedHeadOpKeyGroupStateHandle.openInputStream()) {
		for (int groupId : expectedHeadOpKeyGroupStateHandle.getKeyGroupRange()) {
			long offset = expectedHeadOpKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
			inputStream.seek(offset);
			int expectedKeyGroupState =
					InstantiationUtil.deserializeObject(inputStream, Thread.currentThread().getContextClassLoader());
			for (KeyedStateHandle oneActualKeyedStateHandle : actualPartitionedKeyGroupState) {

				assertTrue(oneActualKeyedStateHandle instanceof KeyGroupsStateHandle);

				KeyGroupsStateHandle oneActualKeyGroupStateHandle = (KeyGroupsStateHandle) oneActualKeyedStateHandle;
				if (oneActualKeyGroupStateHandle.getKeyGroupRange().contains(groupId)) {
					long actualOffset = oneActualKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
					try (FSDataInputStream actualInputStream = oneActualKeyGroupStateHandle.openInputStream()) {
						actualInputStream.seek(actualOffset);
						int actualGroupState = InstantiationUtil.
								deserializeObject(actualInputStream, Thread.currentThread().getContextClassLoader());
						assertEquals(expectedKeyGroupState, actualGroupState);
					}
				}
			}
		}
	}
}

Example 2

Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0

5 votes

public static void compareKeyedState(
		Collection<KeyGroupsStateHandle> expectPartitionedKeyGroupState,
		Collection<? extends KeyedStateHandle> actualPartitionedKeyGroupState) throws Exception {

	KeyGroupsStateHandle expectedHeadOpKeyGroupStateHandle = expectPartitionedKeyGroupState.iterator().next();
	int expectedTotalKeyGroups = expectedHeadOpKeyGroupStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	int actualTotalKeyGroups = 0;
	for(KeyedStateHandle keyedStateHandle: actualPartitionedKeyGroupState) {
		assertTrue(keyedStateHandle instanceof KeyGroupsStateHandle);

		actualTotalKeyGroups += keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	}

	assertEquals(expectedTotalKeyGroups, actualTotalKeyGroups);

	try (FSDataInputStream inputStream = expectedHeadOpKeyGroupStateHandle.openInputStream()) {
		for (int groupId : expectedHeadOpKeyGroupStateHandle.getKeyGroupRange()) {
			long offset = expectedHeadOpKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
			inputStream.seek(offset);
			int expectedKeyGroupState =
					InstantiationUtil.deserializeObject(inputStream, Thread.currentThread().getContextClassLoader());
			for (KeyedStateHandle oneActualKeyedStateHandle : actualPartitionedKeyGroupState) {

				assertTrue(oneActualKeyedStateHandle instanceof KeyGroupsStateHandle);

				KeyGroupsStateHandle oneActualKeyGroupStateHandle = (KeyGroupsStateHandle) oneActualKeyedStateHandle;
				if (oneActualKeyGroupStateHandle.getKeyGroupRange().contains(groupId)) {
					long actualOffset = oneActualKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
					try (FSDataInputStream actualInputStream = oneActualKeyGroupStateHandle.openInputStream()) {
						actualInputStream.seek(actualOffset);
						int actualGroupState = InstantiationUtil.
								deserializeObject(actualInputStream, Thread.currentThread().getContextClassLoader());
						assertEquals(expectedKeyGroupState, actualGroupState);
					}
				}
			}
		}
	}
}

Example 3

Source File: CheckpointCoordinatorTestingUtils.java From flink with Apache License 2.0

5 votes

static void compareKeyedState(
	Collection<KeyGroupsStateHandle> expectPartitionedKeyGroupState,
	Collection<? extends KeyedStateHandle> actualPartitionedKeyGroupState) throws Exception {

	KeyGroupsStateHandle expectedHeadOpKeyGroupStateHandle = expectPartitionedKeyGroupState.iterator().next();
	int expectedTotalKeyGroups = expectedHeadOpKeyGroupStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	int actualTotalKeyGroups = 0;
	for (KeyedStateHandle keyedStateHandle: actualPartitionedKeyGroupState) {
		assertTrue(keyedStateHandle instanceof KeyGroupsStateHandle);

		actualTotalKeyGroups += keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	}

	assertEquals(expectedTotalKeyGroups, actualTotalKeyGroups);

	try (FSDataInputStream inputStream = expectedHeadOpKeyGroupStateHandle.openInputStream()) {
		for (int groupId : expectedHeadOpKeyGroupStateHandle.getKeyGroupRange()) {
			long offset = expectedHeadOpKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
			inputStream.seek(offset);
			int expectedKeyGroupState =
				InstantiationUtil.deserializeObject(inputStream, Thread.currentThread().getContextClassLoader());
			for (KeyedStateHandle oneActualKeyedStateHandle : actualPartitionedKeyGroupState) {

				assertTrue(oneActualKeyedStateHandle instanceof KeyGroupsStateHandle);

				KeyGroupsStateHandle oneActualKeyGroupStateHandle = (KeyGroupsStateHandle) oneActualKeyedStateHandle;
				if (oneActualKeyGroupStateHandle.getKeyGroupRange().contains(groupId)) {
					long actualOffset = oneActualKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
					try (FSDataInputStream actualInputStream = oneActualKeyGroupStateHandle.openInputStream()) {
						actualInputStream.seek(actualOffset);
						int actualGroupState = InstantiationUtil.
							deserializeObject(actualInputStream, Thread.currentThread().getContextClassLoader());
						assertEquals(expectedKeyGroupState, actualGroupState);
					}
				}
			}
		}
	}
}

Example 4

Source File: HeapRestoreOperation.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public Void restore() throws Exception {

	final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();
	registeredKVStates.clear();
	registeredPQStates.clear();

	boolean keySerializerRestored = false;

	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {

		if (keyedStateHandle == null) {
			continue;
		}

		if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected: " + KeyGroupsStateHandle.class +
				", but found: " + keyedStateHandle.getClass());
		}

		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
		FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(fsDataInputStream);

		try {
			DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream);

			KeyedBackendSerializationProxy<K> serializationProxy =
				new KeyedBackendSerializationProxy<>(userCodeClassLoader);

			serializationProxy.read(inView);

			if (!keySerializerRestored) {
				// check for key serializer compatibility; this also reconfigures the
				// key serializer to be compatible, if it is required and is possible
				TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
					keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
				if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
					throw new StateMigrationException("The new key serializer must be compatible.");
				}

				keySerializerRestored = true;
			}

			List<StateMetaInfoSnapshot> restoredMetaInfos =
				serializationProxy.getStateMetaInfoSnapshots();

			createOrCheckStateForMetaInfo(restoredMetaInfos, kvStatesById);

			readStateHandleStateData(
				fsDataInputStream,
				inView,
				keyGroupsStateHandle.getGroupRangeOffsets(),
				kvStatesById, restoredMetaInfos.size(),
				serializationProxy.getReadVersion(),
				serializationProxy.isUsingKeyGroupCompression());
		} finally {
			if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) {
				IOUtils.closeQuietly(fsDataInputStream);
			}
		}
	}
	return null;
}

Example 5

Source File: HeapRestoreOperation.java From flink with Apache License 2.0

4 votes

@Override
public Void restore() throws Exception {

	final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();
	registeredKVStates.clear();
	registeredPQStates.clear();

	boolean keySerializerRestored = false;

	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {

		if (keyedStateHandle == null) {
			continue;
		}

		if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected: " + KeyGroupsStateHandle.class +
				", but found: " + keyedStateHandle.getClass());
		}

		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
		FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(fsDataInputStream);

		try {
			DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream);

			KeyedBackendSerializationProxy<K> serializationProxy =
				new KeyedBackendSerializationProxy<>(userCodeClassLoader);

			serializationProxy.read(inView);

			if (!keySerializerRestored) {
				// check for key serializer compatibility; this also reconfigures the
				// key serializer to be compatible, if it is required and is possible
				TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
					keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
				if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
					throw new StateMigrationException("The new key serializer must be compatible.");
				}

				keySerializerRestored = true;
			}

			List<StateMetaInfoSnapshot> restoredMetaInfos =
				serializationProxy.getStateMetaInfoSnapshots();

			createOrCheckStateForMetaInfo(restoredMetaInfos, kvStatesById);

			readStateHandleStateData(
				fsDataInputStream,
				inView,
				keyGroupsStateHandle.getGroupRangeOffsets(),
				kvStatesById, restoredMetaInfos.size(),
				serializationProxy.getReadVersion(),
				serializationProxy.isUsingKeyGroupCompression());
		} finally {
			if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) {
				IOUtils.closeQuietly(fsDataInputStream);
			}
		}
	}
	return null;
}

Example 6

Source File: HeapRestoreOperation.java From flink with Apache License 2.0

4 votes

@Override
public Void restore() throws Exception {

	registeredKVStates.clear();
	registeredPQStates.clear();

	boolean keySerializerRestored = false;

	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {

		if (keyedStateHandle == null) {
			continue;
		}

		if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected: " + KeyGroupsStateHandle.class +
				", but found: " + keyedStateHandle.getClass());
		}

		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
		FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(fsDataInputStream);

		try {
			DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream);

			KeyedBackendSerializationProxy<K> serializationProxy =
				new KeyedBackendSerializationProxy<>(userCodeClassLoader);

			serializationProxy.read(inView);

			if (!keySerializerRestored) {
				// check for key serializer compatibility; this also reconfigures the
				// key serializer to be compatible, if it is required and is possible
				TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
					keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
				if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
					throw new StateMigrationException("The new key serializer must be compatible.");
				}

				keySerializerRestored = true;
			}

			List<StateMetaInfoSnapshot> restoredMetaInfos =
				serializationProxy.getStateMetaInfoSnapshots();

			final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();

			createOrCheckStateForMetaInfo(restoredMetaInfos, kvStatesById);

			readStateHandleStateData(
				fsDataInputStream,
				inView,
				keyGroupsStateHandle.getGroupRangeOffsets(),
				kvStatesById, restoredMetaInfos.size(),
				serializationProxy.getReadVersion(),
				serializationProxy.isUsingKeyGroupCompression());
		} finally {
			if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) {
				IOUtils.closeQuietly(fsDataInputStream);
			}
		}
	}
	return null;
}