org.apache.flink.runtime.state.ChainedStateHandle Java Examples

The following examples show how to use org.apache.flink.runtime.state.ChainedStateHandle. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CheckpointCoordinatorTestingUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static void verifyStateRestore(
	JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
	List<KeyGroupRange> keyGroupPartitions) throws Exception {

	for (int i = 0; i < executionJobVertex.getParallelism(); i++) {

		JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
		Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
		TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();

		OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));

		ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
			generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);

		assertTrue(CommonTestUtils.isStreamContentEqual(
			expectedOpStateBackend.get(0).openInputStream(),
			operatorState.getManagedOperatorState().iterator().next().openInputStream()));

		KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
			jobVertexID, keyGroupPartitions.get(i), false);
		compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
	}
}
Example #2
Source File: CheckpointCoordinatorTestingUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
static ChainedStateHandle<OperatorStateHandle> generateChainedPartitionableStateHandle(
	JobVertexID jobVertexID,
	int index,
	int namedStates,
	int partitionsPerState,
	boolean rawState) throws IOException {

	Map<String, List<? extends Serializable>> statesListsMap = new HashMap<>(namedStates);

	for (int i = 0; i < namedStates; ++i) {
		List<Integer> testStatesLists = new ArrayList<>(partitionsPerState);
		// generate state
		int seed = jobVertexID.hashCode() * index + i * namedStates;
		if (rawState) {
			seed = (seed + 1) * 31;
		}
		Random random = new Random(seed);
		for (int j = 0; j < partitionsPerState; ++j) {
			int simulatedStateValue = random.nextInt();
			testStatesLists.add(simulatedStateValue);
		}
		statesListsMap.put("state-" + i, testStatesLists);
	}

	return ChainedStateHandle.wrapSingleHandle(generatePartitionableStateHandle(statesListsMap));
}
Example #3
Source File: CheckpointCoordinatorTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public static ChainedStateHandle<OperatorStateHandle> generateChainedPartitionableStateHandle(
		JobVertexID jobVertexID,
		int index,
		int namedStates,
		int partitionsPerState,
		boolean rawState) throws IOException {

	Map<String, List<? extends Serializable>> statesListsMap = new HashMap<>(namedStates);

	for (int i = 0; i < namedStates; ++i) {
		List<Integer> testStatesLists = new ArrayList<>(partitionsPerState);
		// generate state
		int seed = jobVertexID.hashCode() * index + i * namedStates;
		if (rawState) {
			seed = (seed + 1) * 31;
		}
		Random random = new Random(seed);
		for (int j = 0; j < partitionsPerState; ++j) {
			int simulatedStateValue = random.nextInt();
			testStatesLists.add(simulatedStateValue);
		}
		statesListsMap.put("state-" + i, testStatesLists);
	}

	return ChainedStateHandle.wrapSingleHandle(generatePartitionableStateHandle(statesListsMap));
}
Example #4
Source File: CheckpointCoordinatorTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public static void verifyStateRestore(
		JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
		List<KeyGroupRange> keyGroupPartitions) throws Exception {

	for (int i = 0; i < executionJobVertex.getParallelism(); i++) {

		JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
		Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
		TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();

		OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));

		ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
				generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);

		assertTrue(CommonTestUtils.isSteamContentEqual(
				expectedOpStateBackend.get(0).openInputStream(),
				operatorState.getManagedOperatorState().iterator().next().openInputStream()));

		KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
				jobVertexID, keyGroupPartitions.get(i), false);
		compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
	}
}
Example #5
Source File: SubtaskState.java    From flink with Apache License 2.0 6 votes vote down vote up 
public SubtaskState(
		ChainedStateHandle<OperatorStateHandle> managedOperatorState,
		ChainedStateHandle<OperatorStateHandle> rawOperatorState,
		KeyedStateHandle managedKeyedState,
		KeyedStateHandle rawKeyedState) {

	this.managedOperatorState = managedOperatorState;
	this.rawOperatorState = rawOperatorState;
	this.managedKeyedState = managedKeyedState;
	this.rawKeyedState = rawKeyedState;

	try {
		long calculateStateSize = getSizeNullSafe(managedOperatorState);
		calculateStateSize += getSizeNullSafe(rawOperatorState);
		calculateStateSize += getSizeNullSafe(managedKeyedState);
		calculateStateSize += getSizeNullSafe(rawKeyedState);
		stateSize = calculateStateSize;
	} catch (Exception e) {
		throw new RuntimeException("Failed to get state size.", e);
	}
}
Example #6
Source File: SubtaskState.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public SubtaskState(
		ChainedStateHandle<OperatorStateHandle> managedOperatorState,
		ChainedStateHandle<OperatorStateHandle> rawOperatorState,
		KeyedStateHandle managedKeyedState,
		KeyedStateHandle rawKeyedState) {

	this.managedOperatorState = managedOperatorState;
	this.rawOperatorState = rawOperatorState;
	this.managedKeyedState = managedKeyedState;
	this.rawKeyedState = rawKeyedState;

	try {
		long calculateStateSize = getSizeNullSafe(managedOperatorState);
		calculateStateSize += getSizeNullSafe(rawOperatorState);
		calculateStateSize += getSizeNullSafe(managedKeyedState);
		calculateStateSize += getSizeNullSafe(rawKeyedState);
		stateSize = calculateStateSize;
	} catch (Exception e) {
		throw new RuntimeException("Failed to get state size.", e);
	}
}
Example #7
Source File: SubtaskState.java    From flink with Apache License 2.0 6 votes vote down vote up 
public SubtaskState(
		ChainedStateHandle<OperatorStateHandle> managedOperatorState,
		ChainedStateHandle<OperatorStateHandle> rawOperatorState,
		KeyedStateHandle managedKeyedState,
		KeyedStateHandle rawKeyedState) {

	this.managedOperatorState = managedOperatorState;
	this.rawOperatorState = rawOperatorState;
	this.managedKeyedState = managedKeyedState;
	this.rawKeyedState = rawKeyedState;

	try {
		long calculateStateSize = getSizeNullSafe(managedOperatorState);
		calculateStateSize += getSizeNullSafe(rawOperatorState);
		calculateStateSize += getSizeNullSafe(managedKeyedState);
		calculateStateSize += getSizeNullSafe(rawKeyedState);
		stateSize = calculateStateSize;
	} catch (Exception e) {
		throw new RuntimeException("Failed to get state size.", e);
	}
}
Example #8
Source File: CheckpointCoordinatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static ChainedStateHandle<OperatorStateHandle> generateChainedPartitionableStateHandle(
		JobVertexID jobVertexID,
		int index,
		int namedStates,
		int partitionsPerState,
		boolean rawState) throws IOException {

	Map<String, List<? extends Serializable>> statesListsMap = new HashMap<>(namedStates);

	for (int i = 0; i < namedStates; ++i) {
		List<Integer> testStatesLists = new ArrayList<>(partitionsPerState);
		// generate state
		int seed = jobVertexID.hashCode() * index + i * namedStates;
		if (rawState) {
			seed = (seed + 1) * 31;
		}
		Random random = new Random(seed);
		for (int j = 0; j < partitionsPerState; ++j) {
			int simulatedStateValue = random.nextInt();
			testStatesLists.add(simulatedStateValue);
		}
		statesListsMap.put("state-" + i, testStatesLists);
	}

	return ChainedStateHandle.wrapSingleHandle(generatePartitionableStateHandle(statesListsMap));
}
Example #9
Source File: CheckpointCoordinatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static void verifyStateRestore(
		JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
		List<KeyGroupRange> keyGroupPartitions) throws Exception {

	for (int i = 0; i < executionJobVertex.getParallelism(); i++) {

		JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
		Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
		TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();

		OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));

		ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
				generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);

		assertTrue(CommonTestUtils.isStreamContentEqual(
				expectedOpStateBackend.get(0).openInputStream(),
				operatorState.getManagedOperatorState().iterator().next().openInputStream()));

		KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
				jobVertexID, keyGroupPartitions.get(i), false);
		compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
	}
}
Example #10
Source File: SubtaskState.java    From flink with Apache License 2.0 4 votes vote down vote up 
public ChainedStateHandle<OperatorStateHandle> getRawOperatorState() {
	return rawOperatorState;
}
Example #11
Source File: SubtaskState.java    From flink with Apache License 2.0 4 votes vote down vote up 
public ChainedStateHandle<OperatorStateHandle> getRawOperatorState() {
	return rawOperatorState;
}
Example #12
Source File: SubtaskState.java    From flink with Apache License 2.0 4 votes vote down vote up 
public ChainedStateHandle<OperatorStateHandle> getManagedOperatorState() {
	return managedOperatorState;
}
Example #13
Source File: CheckpointTestUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Creates a random collection of TaskState objects containing various types of state handles.
 */
public static Collection<TaskState> createTaskStates(
		Random random,
		int numTaskStates,
		int numSubtasksPerTask) {

	List<TaskState> taskStates = new ArrayList<>(numTaskStates);

	for (int stateIdx = 0; stateIdx < numTaskStates; ++stateIdx) {

		int chainLength = 1 + random.nextInt(8);

		TaskState taskState = new TaskState(new JobVertexID(), numSubtasksPerTask, 128, chainLength);

		int noNonPartitionableStateAtIndex = random.nextInt(chainLength);
		int noOperatorStateBackendAtIndex = random.nextInt(chainLength);
		int noOperatorStateStreamAtIndex = random.nextInt(chainLength);

		boolean hasKeyedBackend = random.nextInt(4) != 0;
		boolean hasKeyedStream = random.nextInt(4) != 0;

		for (int subtaskIdx = 0; subtaskIdx < numSubtasksPerTask; subtaskIdx++) {

			List<OperatorStateHandle> operatorStatesBackend = new ArrayList<>(chainLength);
			List<OperatorStateHandle> operatorStatesStream = new ArrayList<>(chainLength);

			for (int chainIdx = 0; chainIdx < chainLength; ++chainIdx) {

				StreamStateHandle operatorStateBackend =
						new ByteStreamStateHandle("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
				StreamStateHandle operatorStateStream =
						new ByteStreamStateHandle("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
				Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>();
				offsetsMap.put("A", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
				offsetsMap.put("B", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
				offsetsMap.put("C", new OperatorStateHandle.StateMetaInfo(new long[]{60, 70, 80}, OperatorStateHandle.Mode.UNION));

				if (chainIdx != noOperatorStateBackendAtIndex) {
					OperatorStateHandle operatorStateHandleBackend =
							new OperatorStreamStateHandle(offsetsMap, operatorStateBackend);
					operatorStatesBackend.add(operatorStateHandleBackend);
				}

				if (chainIdx != noOperatorStateStreamAtIndex) {
					OperatorStateHandle operatorStateHandleStream =
							new OperatorStreamStateHandle(offsetsMap, operatorStateStream);
					operatorStatesStream.add(operatorStateHandleStream);
				}
			}

			KeyGroupsStateHandle keyedStateBackend = null;
			KeyGroupsStateHandle keyedStateStream = null;

			if (hasKeyedBackend) {
				keyedStateBackend = createDummyKeyGroupStateHandle(random);
			}

			if (hasKeyedStream) {
				keyedStateStream = createDummyKeyGroupStateHandle(random);
			}

			taskState.putState(subtaskIdx, new SubtaskState(
					new ChainedStateHandle<>(operatorStatesBackend),
					new ChainedStateHandle<>(operatorStatesStream),
					keyedStateStream,
					keyedStateBackend));
		}

		taskStates.add(taskState);
	}

	return taskStates;
}
Example #14
Source File: SubtaskState.java    From flink with Apache License 2.0 4 votes vote down vote up 
public ChainedStateHandle<OperatorStateHandle> getManagedOperatorState() {
	return managedOperatorState;
}
Example #15
Source File: CheckpointTestUtils.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Creates a random collection of TaskState objects containing various types of state handles.
 */
public static Collection<TaskState> createTaskStates(
		Random random,
		int numTaskStates,
		int numSubtasksPerTask) {

	List<TaskState> taskStates = new ArrayList<>(numTaskStates);

	for (int stateIdx = 0; stateIdx < numTaskStates; ++stateIdx) {

		int chainLength = 1 + random.nextInt(8);

		TaskState taskState = new TaskState(new JobVertexID(), numSubtasksPerTask, 128, chainLength);

		int noNonPartitionableStateAtIndex = random.nextInt(chainLength);
		int noOperatorStateBackendAtIndex = random.nextInt(chainLength);
		int noOperatorStateStreamAtIndex = random.nextInt(chainLength);

		boolean hasKeyedBackend = random.nextInt(4) != 0;
		boolean hasKeyedStream = random.nextInt(4) != 0;

		for (int subtaskIdx = 0; subtaskIdx < numSubtasksPerTask; subtaskIdx++) {

			List<OperatorStateHandle> operatorStatesBackend = new ArrayList<>(chainLength);
			List<OperatorStateHandle> operatorStatesStream = new ArrayList<>(chainLength);

			for (int chainIdx = 0; chainIdx < chainLength; ++chainIdx) {

				StreamStateHandle operatorStateBackend =
						new ByteStreamStateHandle("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
				StreamStateHandle operatorStateStream =
						new ByteStreamStateHandle("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
				Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>();
				offsetsMap.put("A", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
				offsetsMap.put("B", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
				offsetsMap.put("C", new OperatorStateHandle.StateMetaInfo(new long[]{60, 70, 80}, OperatorStateHandle.Mode.UNION));

				if (chainIdx != noOperatorStateBackendAtIndex) {
					OperatorStateHandle operatorStateHandleBackend =
							new OperatorStreamStateHandle(offsetsMap, operatorStateBackend);
					operatorStatesBackend.add(operatorStateHandleBackend);
				}

				if (chainIdx != noOperatorStateStreamAtIndex) {
					OperatorStateHandle operatorStateHandleStream =
							new OperatorStreamStateHandle(offsetsMap, operatorStateStream);
					operatorStatesStream.add(operatorStateHandleStream);
				}
			}

			KeyGroupsStateHandle keyedStateBackend = null;
			KeyGroupsStateHandle keyedStateStream = null;

			if (hasKeyedBackend) {
				keyedStateBackend = createDummyKeyGroupStateHandle(random);
			}

			if (hasKeyedStream) {
				keyedStateStream = createDummyKeyGroupStateHandle(random);
			}

			taskState.putState(subtaskIdx, new SubtaskState(
					new ChainedStateHandle<>(operatorStatesBackend),
					new ChainedStateHandle<>(operatorStatesStream),
					keyedStateStream,
					keyedStateBackend));
		}

		taskStates.add(taskState);
	}

	return taskStates;
}
Example #16
Source File: SubtaskState.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public ChainedStateHandle<OperatorStateHandle> getRawOperatorState() {
	return rawOperatorState;
}
Example #17
Source File: SubtaskState.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public ChainedStateHandle<OperatorStateHandle> getManagedOperatorState() {
	return managedOperatorState;
}