Java Code Examples for org.apache.flink.runtime.state.OperatorStateHandle#StateMetaInfo

The following examples show how to use org.apache.flink.runtime.state.OperatorStateHandle#StateMetaInfo . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StateAssignmentOperationTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRepartitionSplitDistributeStates() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(1);
	metaInfoMap1.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[30]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(1);
	metaInfoMap2.put("t-2", new OperatorStateHandle.StateMetaInfo(new long[]{0, 15}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[40]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}
Example 2
Source File: StateAssignmentOperationTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRepartitionUnionState() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
	metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{22, 44}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[50]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(1);
	metaInfoMap2.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[20]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}
Example 3
Source File: MetadataV2V3SerializerBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
void serializeOperatorStateHandle(OperatorStateHandle stateHandle, DataOutputStream dos) throws IOException {
	if (stateHandle != null) {
		dos.writeByte(PARTITIONABLE_OPERATOR_STATE_HANDLE);
		Map<String, OperatorStateHandle.StateMetaInfo> partitionOffsetsMap =
				stateHandle.getStateNameToPartitionOffsets();
		dos.writeInt(partitionOffsetsMap.size());
		for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : partitionOffsetsMap.entrySet()) {
			dos.writeUTF(entry.getKey());

			OperatorStateHandle.StateMetaInfo stateMetaInfo = entry.getValue();

			int mode = stateMetaInfo.getDistributionMode().ordinal();
			dos.writeByte(mode);

			long[] offsets = stateMetaInfo.getOffsets();
			dos.writeInt(offsets.length);
			for (long offset : offsets) {
				dos.writeLong(offset);
			}
		}
		serializeStreamStateHandle(stateHandle.getDelegateStateHandle(), dos);
	} else {
		dos.writeByte(NULL_HANDLE);
	}
}
Example 4
Source File: StateHandleDummyUtil.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Creates a deep copy of the given {@link OperatorStreamStateHandle}.
 */
public static OperatorStateHandle deepDummyCopy(OperatorStateHandle original) {

	if (original == null) {
		return null;
	}

	ByteStreamStateHandle stateHandleCopy = cloneByteStreamStateHandle((ByteStreamStateHandle) original.getDelegateStateHandle());
	Map<String, OperatorStateHandle.StateMetaInfo> offsets = original.getStateNameToPartitionOffsets();
	Map<String, OperatorStateHandle.StateMetaInfo> offsetsCopy = new HashMap<>(offsets.size());

	for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : offsets.entrySet()) {
		OperatorStateHandle.StateMetaInfo metaInfo = entry.getValue();
		OperatorStateHandle.StateMetaInfo metaInfoCopy =
			new OperatorStateHandle.StateMetaInfo(metaInfo.getOffsets(), metaInfo.getDistributionMode());
		offsetsCopy.put(String.valueOf(entry.getKey()), metaInfoCopy);
	}
	return new OperatorStreamStateHandle(offsetsCopy, stateHandleCopy);
}
Example 5
Source File: StateAssignmentOperationTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRepartitionUnionState() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(2);
	metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{22, 44}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[50]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(1);
	metaInfoMap2.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[20]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null));

	verifyOneKindPartitionableStateRescale(operatorState, operatorID);
}
Example 6
Source File: SavepointV1Serializer.java    From flink with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
public static OperatorStateHandle deserializeOperatorStateHandle(
		DataInputStream dis) throws IOException {

	final int type = dis.readByte();
	if (NULL_HANDLE == type) {
		return null;
	} else if (PARTITIONABLE_OPERATOR_STATE_HANDLE == type) {
		int mapSize = dis.readInt();
		Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>(mapSize);
		for (int i = 0; i < mapSize; ++i) {
			String key = dis.readUTF();

			int modeOrdinal = dis.readByte();
			OperatorStateHandle.Mode mode = OperatorStateHandle.Mode.values()[modeOrdinal];

			long[] offsets = new long[dis.readInt()];
			for (int j = 0; j < offsets.length; ++j) {
				offsets[j] = dis.readLong();
			}

			OperatorStateHandle.StateMetaInfo metaInfo =
					new OperatorStateHandle.StateMetaInfo(offsets, mode);
			offsetsMap.put(key, metaInfo);
		}
		StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
		return new OperatorStreamStateHandle(offsetsMap, stateHandle);
	} else {
		throw new IllegalStateException("Reading invalid OperatorStateHandle, type: " + type);
	}
}
Example 7
Source File: SavepointV2Serializer.java    From flink with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
public static void serializeOperatorStateHandle(
	OperatorStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle != null) {
		dos.writeByte(PARTITIONABLE_OPERATOR_STATE_HANDLE);
		Map<String, OperatorStateHandle.StateMetaInfo> partitionOffsetsMap =
				stateHandle.getStateNameToPartitionOffsets();
		dos.writeInt(partitionOffsetsMap.size());
		for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : partitionOffsetsMap.entrySet()) {
			dos.writeUTF(entry.getKey());

			OperatorStateHandle.StateMetaInfo stateMetaInfo = entry.getValue();

			int mode = stateMetaInfo.getDistributionMode().ordinal();
			dos.writeByte(mode);

			long[] offsets = stateMetaInfo.getOffsets();
			dos.writeInt(offsets.length);
			for (long offset : offsets) {
				dos.writeLong(offset);
			}
		}
		serializeStreamStateHandle(stateHandle.getDelegateStateHandle(), dos);
	} else {
		dos.writeByte(NULL_HANDLE);
	}
}
Example 8
Source File: StreamTaskStateInitializerImpl.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public boolean hasNext() {

	if (null != offsets && offPos < offsets.length) {

		return true;
	}

	closeCurrentStream();

	while (stateHandleIterator.hasNext()) {
		currentStateHandle = stateHandleIterator.next();
		OperatorStateHandle.StateMetaInfo metaInfo =
			currentStateHandle.getStateNameToPartitionOffsets().get(stateName);

		if (null != metaInfo) {
			long[] metaOffsets = metaInfo.getOffsets();
			if (null != metaOffsets && metaOffsets.length > 0) {
				this.offsets = metaOffsets;
				this.offPos = 0;

				if (closableRegistry.unregisterCloseable(currentStream)) {
					IOUtils.closeQuietly(currentStream);
					currentStream = null;
				}

				return true;
			}
		}
	}

	return false;
}
Example 9
Source File: RoundRobinOperatorStateRepartitioner.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Collect union states from given parallelSubtaskStates.
 */
private Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> collectUnionStates(
	List<List<OperatorStateHandle>> parallelSubtaskStates) {

	Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> unionStates =
		new HashMap<>(parallelSubtaskStates.size());

	for (List<OperatorStateHandle> subTaskState : parallelSubtaskStates) {
		for (OperatorStateHandle operatorStateHandle : subTaskState) {
			if (operatorStateHandle == null) {
				continue;
			}

			final Set<Map.Entry<String, OperatorStateHandle.StateMetaInfo>> partitionOffsetEntries =
				operatorStateHandle.getStateNameToPartitionOffsets().entrySet();

			partitionOffsetEntries.stream()
				.filter(entry -> entry.getValue().getDistributionMode().equals(OperatorStateHandle.Mode.UNION))
				.forEach(entry -> {
					List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>> stateLocations =
						unionStates.computeIfAbsent(entry.getKey(), k -> new ArrayList<>(parallelSubtaskStates.size() * partitionOffsetEntries.size()));

					stateLocations.add(Tuple2.of(operatorStateHandle.getDelegateStateHandle(), entry.getValue()));
				});
		}
	}

	return unionStates;
}
Example 10
Source File: StreamTaskStateInitializerImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public boolean hasNext() {

	if (null != offsets && offPos < offsets.length) {

		return true;
	}

	closeCurrentStream();

	while (stateHandleIterator.hasNext()) {
		currentStateHandle = stateHandleIterator.next();
		OperatorStateHandle.StateMetaInfo metaInfo =
			currentStateHandle.getStateNameToPartitionOffsets().get(stateName);

		if (null != metaInfo) {
			long[] metaOffsets = metaInfo.getOffsets();
			if (null != metaOffsets && metaOffsets.length > 0) {
				this.offsets = metaOffsets;
				this.offPos = 0;

				if (closableRegistry.unregisterCloseable(currentStream)) {
					IOUtils.closeQuietly(currentStream);
					currentStream = null;
				}

				return true;
			}
		}
	}

	return false;
}
Example 11
Source File: CheckpointCoordinatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static void collectResult(int opIdx, OperatorStateHandle operatorStateHandle, List<String> resultCollector) throws Exception {
	try (FSDataInputStream in = operatorStateHandle.openInputStream()) {
		for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : operatorStateHandle.getStateNameToPartitionOffsets().entrySet()) {
			for (long offset : entry.getValue().getOffsets()) {
				in.seek(offset);
				Integer state = InstantiationUtil.
						deserializeObject(in, Thread.currentThread().getContextClassLoader());
				resultCollector.add(opIdx + " : " + entry.getKey() + " : " + state);
			}
		}
	}
}
Example 12
Source File: RoundRobinOperatorStateRepartitioner.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Repartition all named states.
 */
private List<Map<StreamStateHandle, OperatorStateHandle>> repartition(
		GroupByStateNameResults nameToStateByMode,
		int newParallelism) {

	// We will use this to merge w.r.t. StreamStateHandles for each parallel subtask inside the maps
	List<Map<StreamStateHandle, OperatorStateHandle>> mergeMapList = new ArrayList<>(newParallelism);

	// Initialize
	for (int i = 0; i < newParallelism; ++i) {
		mergeMapList.add(new HashMap<>());
	}

	// Start with the state handles we distribute round robin by splitting by offsets
	Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> nameToDistributeState =
			nameToStateByMode.getByMode(OperatorStateHandle.Mode.SPLIT_DISTRIBUTE);

	repartitionSplitState(nameToDistributeState, newParallelism, mergeMapList);

	// Now we also add the state handles marked for union to all parallel instances
	Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> nameToUnionState =
			nameToStateByMode.getByMode(OperatorStateHandle.Mode.UNION);

	repartitionUnionState(nameToUnionState, mergeMapList);

	// Now we also add the state handles marked for uniform broadcast to all parallel instances
	Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> nameToBroadcastState =
			nameToStateByMode.getByMode(OperatorStateHandle.Mode.BROADCAST);

	repartitionBroadcastState(nameToBroadcastState, mergeMapList);

	return mergeMapList;
}
Example 13
Source File: CheckpointCoordinatorTestingUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
static void collectResult(int opIdx, OperatorStateHandle operatorStateHandle, List<String> resultCollector) throws Exception {
	try (FSDataInputStream in = operatorStateHandle.openInputStream()) {
		for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : operatorStateHandle.getStateNameToPartitionOffsets().entrySet()) {
			for (long offset : entry.getValue().getOffsets()) {
				in.seek(offset);
				Integer state = InstantiationUtil.
					deserializeObject(in, Thread.currentThread().getContextClassLoader());
				resultCollector.add(opIdx + " : " + entry.getKey() + " : " + state);
			}
		}
	}
}
Example 14
Source File: StateAssignmentOperationTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Verify repartition logic on partitionable states with all modes.
 */
@Test
public void testReDistributeCombinedPartitionableStates() {
	OperatorID operatorID = new OperatorID();
	OperatorState operatorState = new OperatorState(operatorID, 2, 4);

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap1 = new HashMap<>(6);
	metaInfoMap1.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-2", new OperatorStateHandle.StateMetaInfo(new long[]{22, 44}, OperatorStateHandle.Mode.UNION));
	metaInfoMap1.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[]{52, 63}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
	metaInfoMap1.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{67, 74, 75}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap1.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{77, 88, 92}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap1.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{101, 123, 127}, OperatorStateHandle.Mode.BROADCAST));

	OperatorStateHandle osh1 = new OperatorStreamStateHandle(metaInfoMap1, new ByteStreamStateHandle("test1", new byte[130]));
	operatorState.putState(0, new OperatorSubtaskState(osh1, null, null, null, null, null));

	Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap2 = new HashMap<>(3);
	metaInfoMap2.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[]{0}, OperatorStateHandle.Mode.UNION));
	metaInfoMap2.put("t-4", new OperatorStateHandle.StateMetaInfo(new long[]{20, 27, 28}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap2.put("t-5", new OperatorStateHandle.StateMetaInfo(new long[]{30, 44, 48}, OperatorStateHandle.Mode.BROADCAST));
	metaInfoMap2.put("t-6", new OperatorStateHandle.StateMetaInfo(new long[]{57, 79, 83}, OperatorStateHandle.Mode.BROADCAST));

	OperatorStateHandle osh2 = new OperatorStreamStateHandle(metaInfoMap2, new ByteStreamStateHandle("test2", new byte[86]));
	operatorState.putState(1, new OperatorSubtaskState(osh2, null, null, null, null, null));

	// rescale up case, parallelism 2 --> 3
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 3);

	// rescale down case, parallelism 2 --> 1
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 1);

	// not rescale
	verifyCombinedPartitionableStateRescale(operatorState, operatorID, 2, 2);
}
Example 15
Source File: SavepointV1Serializer.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
public static void serializeOperatorStateHandle(
	OperatorStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle != null) {
		dos.writeByte(PARTITIONABLE_OPERATOR_STATE_HANDLE);
		Map<String, OperatorStateHandle.StateMetaInfo> partitionOffsetsMap =
				stateHandle.getStateNameToPartitionOffsets();
		dos.writeInt(partitionOffsetsMap.size());
		for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : partitionOffsetsMap.entrySet()) {
			dos.writeUTF(entry.getKey());

			OperatorStateHandle.StateMetaInfo stateMetaInfo = entry.getValue();

			int mode = stateMetaInfo.getDistributionMode().ordinal();
			dos.writeByte(mode);

			long[] offsets = stateMetaInfo.getOffsets();
			dos.writeInt(offsets.length);
			for (long offset : offsets) {
				dos.writeLong(offset);
			}
		}
		serializeStreamStateHandle(stateHandle.getDelegateStateHandle(), dos);
	} else {
		dos.writeByte(NULL_HANDLE);
	}
}
Example 16
Source File: RoundRobinOperatorStateRepartitioner.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Group by the different named states.
 */
@SuppressWarnings("unchecked, rawtype")
private GroupByStateNameResults groupByStateMode(List<List<OperatorStateHandle>> previousParallelSubtaskStates) {

	//Reorganize: group by (State Name -> StreamStateHandle + StateMetaInfo)
	EnumMap<OperatorStateHandle.Mode,
			Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>>> nameToStateByMode =
			new EnumMap<>(OperatorStateHandle.Mode.class);

	for (OperatorStateHandle.Mode mode : OperatorStateHandle.Mode.values()) {

		nameToStateByMode.put(
				mode,
				new HashMap<>());
	}

	for (List<OperatorStateHandle> previousParallelSubtaskState : previousParallelSubtaskStates) {
		for (OperatorStateHandle operatorStateHandle : previousParallelSubtaskState) {

			if (operatorStateHandle == null) {
				continue;
			}

			final Set<Map.Entry<String, OperatorStateHandle.StateMetaInfo>> partitionOffsetEntries =
				operatorStateHandle.getStateNameToPartitionOffsets().entrySet();

			for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> e : partitionOffsetEntries) {
				OperatorStateHandle.StateMetaInfo metaInfo = e.getValue();

				Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> nameToState =
					nameToStateByMode.get(metaInfo.getDistributionMode());

				List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>> stateLocations =
					nameToState.computeIfAbsent(
						e.getKey(),
						k -> new ArrayList<>(previousParallelSubtaskStates.size() * partitionOffsetEntries.size()));

				stateLocations.add(Tuple2.of(operatorStateHandle.getDelegateStateHandle(), e.getValue()));
			}
		}
	}

	return new GroupByStateNameResults(nameToStateByMode);
}
Example 17
Source File: CheckpointTestUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Creates a random collection of TaskState objects containing various types of state handles.
 */
public static Collection<TaskState> createTaskStates(
		Random random,
		int numTaskStates,
		int numSubtasksPerTask) {

	List<TaskState> taskStates = new ArrayList<>(numTaskStates);

	for (int stateIdx = 0; stateIdx < numTaskStates; ++stateIdx) {

		int chainLength = 1 + random.nextInt(8);

		TaskState taskState = new TaskState(new JobVertexID(), numSubtasksPerTask, 128, chainLength);

		int noNonPartitionableStateAtIndex = random.nextInt(chainLength);
		int noOperatorStateBackendAtIndex = random.nextInt(chainLength);
		int noOperatorStateStreamAtIndex = random.nextInt(chainLength);

		boolean hasKeyedBackend = random.nextInt(4) != 0;
		boolean hasKeyedStream = random.nextInt(4) != 0;

		for (int subtaskIdx = 0; subtaskIdx < numSubtasksPerTask; subtaskIdx++) {

			List<OperatorStateHandle> operatorStatesBackend = new ArrayList<>(chainLength);
			List<OperatorStateHandle> operatorStatesStream = new ArrayList<>(chainLength);

			for (int chainIdx = 0; chainIdx < chainLength; ++chainIdx) {

				StreamStateHandle operatorStateBackend =
						new ByteStreamStateHandle("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
				StreamStateHandle operatorStateStream =
						new ByteStreamStateHandle("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
				Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>();
				offsetsMap.put("A", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
				offsetsMap.put("B", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
				offsetsMap.put("C", new OperatorStateHandle.StateMetaInfo(new long[]{60, 70, 80}, OperatorStateHandle.Mode.UNION));

				if (chainIdx != noOperatorStateBackendAtIndex) {
					OperatorStateHandle operatorStateHandleBackend =
							new OperatorStreamStateHandle(offsetsMap, operatorStateBackend);
					operatorStatesBackend.add(operatorStateHandleBackend);
				}

				if (chainIdx != noOperatorStateStreamAtIndex) {
					OperatorStateHandle operatorStateHandleStream =
							new OperatorStreamStateHandle(offsetsMap, operatorStateStream);
					operatorStatesStream.add(operatorStateHandleStream);
				}
			}

			KeyGroupsStateHandle keyedStateBackend = null;
			KeyGroupsStateHandle keyedStateStream = null;

			if (hasKeyedBackend) {
				keyedStateBackend = createDummyKeyGroupStateHandle(random);
			}

			if (hasKeyedStream) {
				keyedStateStream = createDummyKeyGroupStateHandle(random);
			}

			taskState.putState(subtaskIdx, new SubtaskState(
					new ChainedStateHandle<>(operatorStatesBackend),
					new ChainedStateHandle<>(operatorStatesStream),
					keyedStateStream,
					keyedStateBackend));
		}

		taskStates.add(taskState);
	}

	return taskStates;
}
Example 18
Source File: RoundRobinOperatorStateRepartitioner.java    From flink with Apache License 2.0 4 votes vote down vote up 
GroupByStateNameResults(
		EnumMap<OperatorStateHandle.Mode,
				Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>>> byMode) {
	this.byMode = Preconditions.checkNotNull(byMode);
}
Example 19
Source File: RoundRobinOperatorStateRepartitioner.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public List<List<OperatorStateHandle>> repartitionState(
		List<List<OperatorStateHandle>> previousParallelSubtaskStates,
		int oldParallelism,
		int newParallelism) {

	Preconditions.checkNotNull(previousParallelSubtaskStates);
	Preconditions.checkArgument(newParallelism > 0);
	Preconditions.checkArgument(previousParallelSubtaskStates.size() == oldParallelism,
		"This method still depends on the order of the new and old operators");

	// Assemble result from all merge maps
	List<List<OperatorStateHandle>> result = new ArrayList<>(newParallelism);

	List<Map<StreamStateHandle, OperatorStateHandle>> mergeMapList;

	// We only round-robin repartition UNION state if new parallelism equals to the old one.
	if (newParallelism == oldParallelism) {
		Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> unionStates = collectUnionStates(previousParallelSubtaskStates);

		if (unionStates.isEmpty()) {
			return previousParallelSubtaskStates;
		}

		// Initialize
		mergeMapList = initMergeMapList(previousParallelSubtaskStates);

		repartitionUnionState(unionStates, mergeMapList);
	} else {

		// Reorganize: group by (State Name -> StreamStateHandle + Offsets)
		GroupByStateNameResults nameToStateByMode = groupByStateMode(previousParallelSubtaskStates);

		if (OPTIMIZE_MEMORY_USE) {
			previousParallelSubtaskStates.clear(); // free for GC at to cost that old handles are no longer available
		}

		// Do the actual repartitioning for all named states
		mergeMapList =
			repartition(nameToStateByMode, newParallelism);
	}

	for (int i = 0; i < mergeMapList.size(); ++i) {
		result.add(i, new ArrayList<>(mergeMapList.get(i).values()));
	}

	return result;
}
Example 20
Source File: RoundRobinOperatorStateRepartitioner.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Group by the different named states.
 */
@SuppressWarnings("unchecked, rawtype")
private GroupByStateNameResults groupByStateMode(List<List<OperatorStateHandle>> previousParallelSubtaskStates) {

	//Reorganize: group by (State Name -> StreamStateHandle + StateMetaInfo)
	EnumMap<OperatorStateHandle.Mode,
			Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>>> nameToStateByMode =
			new EnumMap<>(OperatorStateHandle.Mode.class);

	for (OperatorStateHandle.Mode mode : OperatorStateHandle.Mode.values()) {

		nameToStateByMode.put(
				mode,
				new HashMap<>());
	}

	for (List<OperatorStateHandle> previousParallelSubtaskState : previousParallelSubtaskStates) {
		for (OperatorStateHandle operatorStateHandle : previousParallelSubtaskState) {

			if (operatorStateHandle == null) {
				continue;
			}

			final Set<Map.Entry<String, OperatorStateHandle.StateMetaInfo>> partitionOffsetEntries =
				operatorStateHandle.getStateNameToPartitionOffsets().entrySet();

			for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> e : partitionOffsetEntries) {
				OperatorStateHandle.StateMetaInfo metaInfo = e.getValue();

				Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> nameToState =
					nameToStateByMode.get(metaInfo.getDistributionMode());

				List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>> stateLocations =
					nameToState.computeIfAbsent(
						e.getKey(),
						k -> new ArrayList<>(previousParallelSubtaskStates.size() * partitionOffsetEntries.size()));

				stateLocations.add(Tuple2.of(operatorStateHandle.getDelegateStateHandle(), e.getValue()));
			}
		}
	}

	return new GroupByStateNameResults(nameToStateByMode);
}