Java Code Examples for org.apache.flink.runtime.state.KeyGroupRangeAssignment#computeKeyGroupRangeForOperatorIndex()

The following examples show how to use org.apache.flink.runtime.state.KeyGroupRangeAssignment#computeKeyGroupRangeForOperatorIndex() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StreamTaskStateInitializerImpl.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
protected <K> AbstractKeyedStateBackend<K> keyedStatedBackend(
	TypeSerializer<K> keySerializer,
	String operatorIdentifierText,
	PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskStates,
	CloseableRegistry backendCloseableRegistry,
	MetricGroup metricGroup) throws Exception {

	if (keySerializer == null) {
		return null;
	}

	String logDescription = "keyed state backend for " + operatorIdentifierText;

	TaskInfo taskInfo = environment.getTaskInfo();

	final KeyGroupRange keyGroupRange = KeyGroupRangeAssignment.computeKeyGroupRangeForOperatorIndex(
		taskInfo.getMaxNumberOfParallelSubtasks(),
		taskInfo.getNumberOfParallelSubtasks(),
		taskInfo.getIndexOfThisSubtask());

	// Now restore processing is included in backend building/constructing process, so we need to make sure
	// each stream constructed in restore could also be closed in case of task cancel, for example the data
	// input stream opened for serDe during restore.
	CloseableRegistry cancelStreamRegistryForRestore = new CloseableRegistry();
	backendCloseableRegistry.registerCloseable(cancelStreamRegistryForRestore);
	BackendRestorerProcedure<AbstractKeyedStateBackend<K>, KeyedStateHandle> backendRestorer =
		new BackendRestorerProcedure<>(
			(stateHandles) -> stateBackend.createKeyedStateBackend(
				environment,
				environment.getJobID(),
				operatorIdentifierText,
				keySerializer,
				taskInfo.getMaxNumberOfParallelSubtasks(),
				keyGroupRange,
				environment.getTaskKvStateRegistry(),
				TtlTimeProvider.DEFAULT,
				metricGroup,
				stateHandles,
				cancelStreamRegistryForRestore),
			backendCloseableRegistry,
			logDescription);

	try {
		return backendRestorer.createAndRestore(
			prioritizedOperatorSubtaskStates.getPrioritizedManagedKeyedState());
	} finally {
		if (backendCloseableRegistry.unregisterCloseable(cancelStreamRegistryForRestore)) {
			IOUtils.closeQuietly(cancelStreamRegistryForRestore);
		}
	}
}
Example 2
Source File: StreamTaskStateInitializerImpl.java    From flink with Apache License 2.0 4 votes vote down vote up 
protected <K> AbstractKeyedStateBackend<K> keyedStatedBackend(
	TypeSerializer<K> keySerializer,
	String operatorIdentifierText,
	PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskStates,
	CloseableRegistry backendCloseableRegistry,
	MetricGroup metricGroup) throws Exception {

	if (keySerializer == null) {
		return null;
	}

	String logDescription = "keyed state backend for " + operatorIdentifierText;

	TaskInfo taskInfo = environment.getTaskInfo();

	final KeyGroupRange keyGroupRange = KeyGroupRangeAssignment.computeKeyGroupRangeForOperatorIndex(
		taskInfo.getMaxNumberOfParallelSubtasks(),
		taskInfo.getNumberOfParallelSubtasks(),
		taskInfo.getIndexOfThisSubtask());

	// Now restore processing is included in backend building/constructing process, so we need to make sure
	// each stream constructed in restore could also be closed in case of task cancel, for example the data
	// input stream opened for serDe during restore.
	CloseableRegistry cancelStreamRegistryForRestore = new CloseableRegistry();
	backendCloseableRegistry.registerCloseable(cancelStreamRegistryForRestore);
	BackendRestorerProcedure<AbstractKeyedStateBackend<K>, KeyedStateHandle> backendRestorer =
		new BackendRestorerProcedure<>(
			(stateHandles) -> stateBackend.createKeyedStateBackend(
				environment,
				environment.getJobID(),
				operatorIdentifierText,
				keySerializer,
				taskInfo.getMaxNumberOfParallelSubtasks(),
				keyGroupRange,
				environment.getTaskKvStateRegistry(),
				TtlTimeProvider.DEFAULT,
				metricGroup,
				stateHandles,
				cancelStreamRegistryForRestore),
			backendCloseableRegistry,
			logDescription);

	try {
		return backendRestorer.createAndRestore(
			prioritizedOperatorSubtaskStates.getPrioritizedManagedKeyedState());
	} finally {
		if (backendCloseableRegistry.unregisterCloseable(cancelStreamRegistryForRestore)) {
			IOUtils.closeQuietly(cancelStreamRegistryForRestore);
		}
	}
}
Example 3
Source File: StreamTaskStateInitializerImpl.java    From flink with Apache License 2.0 4 votes vote down vote up 
protected <K> AbstractKeyedStateBackend<K> keyedStatedBackend(
	TypeSerializer<K> keySerializer,
	String operatorIdentifierText,
	PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskStates,
	CloseableRegistry backendCloseableRegistry,
	MetricGroup metricGroup) throws Exception {

	if (keySerializer == null) {
		return null;
	}

	String logDescription = "keyed state backend for " + operatorIdentifierText;

	TaskInfo taskInfo = environment.getTaskInfo();

	final KeyGroupRange keyGroupRange = KeyGroupRangeAssignment.computeKeyGroupRangeForOperatorIndex(
		taskInfo.getMaxNumberOfParallelSubtasks(),
		taskInfo.getNumberOfParallelSubtasks(),
		taskInfo.getIndexOfThisSubtask());

	// Now restore processing is included in backend building/constructing process, so we need to make sure
	// each stream constructed in restore could also be closed in case of task cancel, for example the data
	// input stream opened for serDe during restore.
	CloseableRegistry cancelStreamRegistryForRestore = new CloseableRegistry();
	backendCloseableRegistry.registerCloseable(cancelStreamRegistryForRestore);
	BackendRestorerProcedure<AbstractKeyedStateBackend<K>, KeyedStateHandle> backendRestorer =
		new BackendRestorerProcedure<>(
			(stateHandles) -> stateBackend.createKeyedStateBackend(
				environment,
				environment.getJobID(),
				operatorIdentifierText,
				keySerializer,
				taskInfo.getMaxNumberOfParallelSubtasks(),
				keyGroupRange,
				environment.getTaskKvStateRegistry(),
				ttlTimeProvider,
				metricGroup,
				stateHandles,
				cancelStreamRegistryForRestore),
			backendCloseableRegistry,
			logDescription);

	try {
		return backendRestorer.createAndRestore(
			prioritizedOperatorSubtaskStates.getPrioritizedManagedKeyedState());
	} finally {
		if (backendCloseableRegistry.unregisterCloseable(cancelStreamRegistryForRestore)) {
			IOUtils.closeQuietly(cancelStreamRegistryForRestore);
		}
	}
}