Java Code Examples for org.apache.flink.runtime.state.KeyGroupsStateHandle#getKeyGroupRange()

The following examples show how to use org.apache.flink.runtime.state.KeyGroupsStateHandle#getKeyGroupRange() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SavepointV1Serializer.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}
Example 2
Source File: SavepointV1Serializer.java    From flink with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}
Example 3
Source File: SavepointV2Serializer.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) stateHandle;

		dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE);

		dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
		dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());

		serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);

		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}
Example 4
Source File: CheckpointCoordinatorTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public static void compareKeyedState(
		Collection<KeyGroupsStateHandle> expectPartitionedKeyGroupState,
		Collection<? extends KeyedStateHandle> actualPartitionedKeyGroupState) throws Exception {

	KeyGroupsStateHandle expectedHeadOpKeyGroupStateHandle = expectPartitionedKeyGroupState.iterator().next();
	int expectedTotalKeyGroups = expectedHeadOpKeyGroupStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	int actualTotalKeyGroups = 0;
	for(KeyedStateHandle keyedStateHandle: actualPartitionedKeyGroupState) {
		assertTrue(keyedStateHandle instanceof KeyGroupsStateHandle);

		actualTotalKeyGroups += keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	}

	assertEquals(expectedTotalKeyGroups, actualTotalKeyGroups);

	try (FSDataInputStream inputStream = expectedHeadOpKeyGroupStateHandle.openInputStream()) {
		for (int groupId : expectedHeadOpKeyGroupStateHandle.getKeyGroupRange()) {
			long offset = expectedHeadOpKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
			inputStream.seek(offset);
			int expectedKeyGroupState =
					InstantiationUtil.deserializeObject(inputStream, Thread.currentThread().getContextClassLoader());
			for (KeyedStateHandle oneActualKeyedStateHandle : actualPartitionedKeyGroupState) {

				assertTrue(oneActualKeyedStateHandle instanceof KeyGroupsStateHandle);

				KeyGroupsStateHandle oneActualKeyGroupStateHandle = (KeyGroupsStateHandle) oneActualKeyedStateHandle;
				if (oneActualKeyGroupStateHandle.getKeyGroupRange().contains(groupId)) {
					long actualOffset = oneActualKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
					try (FSDataInputStream actualInputStream = oneActualKeyGroupStateHandle.openInputStream()) {
						actualInputStream.seek(actualOffset);
						int actualGroupState = InstantiationUtil.
								deserializeObject(actualInputStream, Thread.currentThread().getContextClassLoader());
						assertEquals(expectedKeyGroupState, actualGroupState);
					}
				}
			}
		}
	}
}
Example 5
Source File: SavepointV2Serializer.java    From flink with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) stateHandle;

		dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE);

		dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
		dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());

		serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);

		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}
Example 6
Source File: CheckpointCoordinatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static void compareKeyedState(
		Collection<KeyGroupsStateHandle> expectPartitionedKeyGroupState,
		Collection<? extends KeyedStateHandle> actualPartitionedKeyGroupState) throws Exception {

	KeyGroupsStateHandle expectedHeadOpKeyGroupStateHandle = expectPartitionedKeyGroupState.iterator().next();
	int expectedTotalKeyGroups = expectedHeadOpKeyGroupStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	int actualTotalKeyGroups = 0;
	for(KeyedStateHandle keyedStateHandle: actualPartitionedKeyGroupState) {
		assertTrue(keyedStateHandle instanceof KeyGroupsStateHandle);

		actualTotalKeyGroups += keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	}

	assertEquals(expectedTotalKeyGroups, actualTotalKeyGroups);

	try (FSDataInputStream inputStream = expectedHeadOpKeyGroupStateHandle.openInputStream()) {
		for (int groupId : expectedHeadOpKeyGroupStateHandle.getKeyGroupRange()) {
			long offset = expectedHeadOpKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
			inputStream.seek(offset);
			int expectedKeyGroupState =
					InstantiationUtil.deserializeObject(inputStream, Thread.currentThread().getContextClassLoader());
			for (KeyedStateHandle oneActualKeyedStateHandle : actualPartitionedKeyGroupState) {

				assertTrue(oneActualKeyedStateHandle instanceof KeyGroupsStateHandle);

				KeyGroupsStateHandle oneActualKeyGroupStateHandle = (KeyGroupsStateHandle) oneActualKeyedStateHandle;
				if (oneActualKeyGroupStateHandle.getKeyGroupRange().contains(groupId)) {
					long actualOffset = oneActualKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
					try (FSDataInputStream actualInputStream = oneActualKeyGroupStateHandle.openInputStream()) {
						actualInputStream.seek(actualOffset);
						int actualGroupState = InstantiationUtil.
								deserializeObject(actualInputStream, Thread.currentThread().getContextClassLoader());
						assertEquals(expectedKeyGroupState, actualGroupState);
					}
				}
			}
		}
	}
}
Example 7
Source File: MetadataV2V3SerializerBase.java    From flink with Apache License 2.0 5 votes vote down vote up 
void serializeKeyedStateHandle(KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {
	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) stateHandle;

		dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE);

		dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
		dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());

		serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);

		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}
Example 8
Source File: CheckpointCoordinatorTestingUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
static void compareKeyedState(
	Collection<KeyGroupsStateHandle> expectPartitionedKeyGroupState,
	Collection<? extends KeyedStateHandle> actualPartitionedKeyGroupState) throws Exception {

	KeyGroupsStateHandle expectedHeadOpKeyGroupStateHandle = expectPartitionedKeyGroupState.iterator().next();
	int expectedTotalKeyGroups = expectedHeadOpKeyGroupStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	int actualTotalKeyGroups = 0;
	for (KeyedStateHandle keyedStateHandle: actualPartitionedKeyGroupState) {
		assertTrue(keyedStateHandle instanceof KeyGroupsStateHandle);

		actualTotalKeyGroups += keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	}

	assertEquals(expectedTotalKeyGroups, actualTotalKeyGroups);

	try (FSDataInputStream inputStream = expectedHeadOpKeyGroupStateHandle.openInputStream()) {
		for (int groupId : expectedHeadOpKeyGroupStateHandle.getKeyGroupRange()) {
			long offset = expectedHeadOpKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
			inputStream.seek(offset);
			int expectedKeyGroupState =
				InstantiationUtil.deserializeObject(inputStream, Thread.currentThread().getContextClassLoader());
			for (KeyedStateHandle oneActualKeyedStateHandle : actualPartitionedKeyGroupState) {

				assertTrue(oneActualKeyedStateHandle instanceof KeyGroupsStateHandle);

				KeyGroupsStateHandle oneActualKeyGroupStateHandle = (KeyGroupsStateHandle) oneActualKeyedStateHandle;
				if (oneActualKeyGroupStateHandle.getKeyGroupRange().contains(groupId)) {
					long actualOffset = oneActualKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
					try (FSDataInputStream actualInputStream = oneActualKeyGroupStateHandle.openInputStream()) {
						actualInputStream.seek(actualOffset);
						int actualGroupState = InstantiationUtil.
							deserializeObject(actualInputStream, Thread.currentThread().getContextClassLoader());
						assertEquals(expectedKeyGroupState, actualGroupState);
					}
				}
			}
		}
	}
}