Java Code Examples for org.apache.flink.runtime.state.StreamCompressionDecorator#decorateWithCompression()

The following examples show how to use org.apache.flink.runtime.state.StreamCompressionDecorator#decorateWithCompression() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HeapRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
Example 2
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
Example 3
Source File: RocksDBSavepointIterator.java    From bravo with Apache License 2.0 5 votes vote down vote up 
private boolean openIfNeeded() throws Exception {
	if (stateHandleInStream == null) {
		LOGGER.debug("Opening {}", keyGroupsStateHandle.getDelegateStateHandle());
		stateHandleInStream = keyGroupsStateHandle.openInputStream();

		final KeyedBackendSerializationProxy<?> serializationProxy = StateMetadataUtils
				.getKeyedBackendSerializationProxy(keyGroupsStateHandle);

		this.stateIdMapping = StateMetadataUtils.getStateIdMapping(serializationProxy);
		final StreamCompressionDecorator streamCompressionDecorator = StateMetadataUtils
				.getCompressionDecorator(serializationProxy);

		final KeyGroupRangeOffsets rangeOffsets = keyGroupsStateHandle.getGroupRangeOffsets();
		LOGGER.debug("{}", rangeOffsets);

		offsetsIt = new ValidOffsetsIterator(rangeOffsets);

		hasNext = seekNextOffset();

		if (hasNext) {
			final InputStream compressedInputStream = streamCompressionDecorator
					.decorateWithCompression(stateHandleInStream);
			compressedInputView = new DataInputViewStreamWrapper(compressedInputStream);
			seekNextStateId(false);
		}
	}

	return hasNext;
}
Example 4
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}