org.apache.flink.runtime.state.KeyGroupRangeOffsets Java Examples

The following examples show how to use org.apache.flink.runtime.state.KeyGroupRangeOffsets. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RocksFullSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private void writeSnapshotToOutputStream(
	@Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider,
	@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {

	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators =
		new ArrayList<>(metaData.size());
	final DataOutputView outputView =
		new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream());
	final ReadOptions readOptions = new ReadOptions();
	try {
		readOptions.setSnapshot(snapshot);
		writeKVStateMetaData(kvStateIterators, readOptions, outputView);
		writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets);
	} finally {

		for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) {
			IOUtils.closeQuietly(kvStateIterator.f0);
		}

		IOUtils.closeQuietly(readOptions);
	}
}
Example #2
Source File: SavepointV1Serializer.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
public static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException {
	final int type = dis.readByte();
	if (NULL_HANDLE == type) {
		return null;
	} else if (KEY_GROUPS_HANDLE == type) {
		int startKeyGroup = dis.readInt();
		int numKeyGroups = dis.readInt();
		KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
		long[] offsets = new long[numKeyGroups];
		for (int i = 0; i < numKeyGroups; ++i) {
			offsets[i] = dis.readLong();
		}
		KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(
			keyGroupRange, offsets);
		StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
		return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle);
	} else {
		throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type);
	}
}
Example #3
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void writeSnapshotToOutputStream(
	@Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider,
	@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {

	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators =
		new ArrayList<>(metaData.size());
	final DataOutputView outputView =
		new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream());
	final ReadOptions readOptions = new ReadOptions();
	try {
		readOptions.setSnapshot(snapshot);
		writeKVStateMetaData(kvStateIterators, readOptions, outputView);
		writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets);
	} finally {

		for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) {
			IOUtils.closeQuietly(kvStateIterator.f0);
		}

		IOUtils.closeQuietly(readOptions);
	}
}
Example #4
Source File: CheckpointCoordinatorTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public static KeyGroupsStateHandle generateKeyGroupState(
		KeyGroupRange keyGroupRange,
		List<? extends Serializable> states) throws IOException {

	Preconditions.checkArgument(keyGroupRange.getNumberOfKeyGroups() == states.size());

	Tuple2<byte[], List<long[]>> serializedDataWithOffsets =
			serializeTogetherAndTrackOffsets(Collections.<List<? extends Serializable>>singletonList(states));

	KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, serializedDataWithOffsets.f1.get(0));

	ByteStreamStateHandle allSerializedStatesHandle = new ByteStreamStateHandle(
			String.valueOf(UUID.randomUUID()),
			serializedDataWithOffsets.f0);

	return new KeyGroupsStateHandle(keyGroupRangeOffsets, allSerializedStatesHandle);
}
Example #5
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
protected SnapshotResult<KeyedStateHandle> callInternal() throws Exception {
	final KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange);
	final CheckpointStreamWithResultProvider checkpointStreamWithResultProvider =
		checkpointStreamSupplier.get();

	snapshotCloseableRegistry.registerCloseable(checkpointStreamWithResultProvider);
	writeSnapshotToOutputStream(checkpointStreamWithResultProvider, keyGroupRangeOffsets);

	if (snapshotCloseableRegistry.unregisterCloseable(checkpointStreamWithResultProvider)) {
		return CheckpointStreamWithResultProvider.toKeyedStateHandleSnapshotResult(
			checkpointStreamWithResultProvider.closeAndFinalizeCheckpointStreamResult(),
			keyGroupRangeOffsets);
	} else {
		throw new IOException("Stream is already unregistered/closed.");
	}
}
Example #6
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void writeSnapshotToOutputStream(
	@Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider,
	@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {

	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators =
		new ArrayList<>(metaData.size());
	final DataOutputView outputView =
		new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream());
	final ReadOptions readOptions = new ReadOptions();
	try {
		readOptions.setSnapshot(snapshot);
		writeKVStateMetaData(kvStateIterators, readOptions, outputView);
		writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets);
	} finally {

		for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) {
			IOUtils.closeQuietly(kvStateIterator.f0);
		}

		IOUtils.closeQuietly(readOptions);
	}
}
Example #7
Source File: SavepointV1Serializer.java    From flink with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
public static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException {
	final int type = dis.readByte();
	if (NULL_HANDLE == type) {
		return null;
	} else if (KEY_GROUPS_HANDLE == type) {
		int startKeyGroup = dis.readInt();
		int numKeyGroups = dis.readInt();
		KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
		long[] offsets = new long[numKeyGroups];
		for (int i = 0; i < numKeyGroups; ++i) {
			offsets[i] = dis.readLong();
		}
		KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(
			keyGroupRange, offsets);
		StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
		return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle);
	} else {
		throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type);
	}
}
Example #8
Source File: RocksFullSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
protected SnapshotResult<KeyedStateHandle> callInternal() throws Exception {
	final KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange);
	final CheckpointStreamWithResultProvider checkpointStreamWithResultProvider =
		checkpointStreamSupplier.get();

	snapshotCloseableRegistry.registerCloseable(checkpointStreamWithResultProvider);
	writeSnapshotToOutputStream(checkpointStreamWithResultProvider, keyGroupRangeOffsets);

	if (snapshotCloseableRegistry.unregisterCloseable(checkpointStreamWithResultProvider)) {
		return CheckpointStreamWithResultProvider.toKeyedStateHandleSnapshotResult(
			checkpointStreamWithResultProvider.closeAndFinalizeCheckpointStreamResult(),
			keyGroupRangeOffsets);
	} else {
		throw new IOException("Stream is already unregistered/closed.");
	}
}
Example #9
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
protected SnapshotResult<KeyedStateHandle> callInternal() throws Exception {
	final KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange);
	final CheckpointStreamWithResultProvider checkpointStreamWithResultProvider =
		checkpointStreamSupplier.get();

	snapshotCloseableRegistry.registerCloseable(checkpointStreamWithResultProvider);
	writeSnapshotToOutputStream(checkpointStreamWithResultProvider, keyGroupRangeOffsets);

	if (snapshotCloseableRegistry.unregisterCloseable(checkpointStreamWithResultProvider)) {
		return CheckpointStreamWithResultProvider.toKeyedStateHandleSnapshotResult(
			checkpointStreamWithResultProvider.closeAndFinalizeCheckpointStreamResult(),
			keyGroupRangeOffsets);
	} else {
		throw new IOException("Stream is already unregistered/closed.");
	}
}
Example #10
Source File: CheckpointCoordinatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static KeyGroupsStateHandle generateKeyGroupState(
		KeyGroupRange keyGroupRange,
		List<? extends Serializable> states) throws IOException {

	Preconditions.checkArgument(keyGroupRange.getNumberOfKeyGroups() == states.size());

	Tuple2<byte[], List<long[]>> serializedDataWithOffsets =
			serializeTogetherAndTrackOffsets(Collections.<List<? extends Serializable>>singletonList(states));

	KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, serializedDataWithOffsets.f1.get(0));

	ByteStreamStateHandle allSerializedStatesHandle = new ByteStreamStateHandle(
			String.valueOf(UUID.randomUUID()),
			serializedDataWithOffsets.f0);

	return new KeyGroupsStateHandle(keyGroupRangeOffsets, allSerializedStatesHandle);
}
Example #11
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
Example #12
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
Example #13
Source File: RocksDBSavepointIterator.java    From bravo with Apache License 2.0 5 votes vote down vote up 
private boolean openIfNeeded() throws Exception {
	if (stateHandleInStream == null) {
		LOGGER.debug("Opening {}", keyGroupsStateHandle.getDelegateStateHandle());
		stateHandleInStream = keyGroupsStateHandle.openInputStream();

		final KeyedBackendSerializationProxy<?> serializationProxy = StateMetadataUtils
				.getKeyedBackendSerializationProxy(keyGroupsStateHandle);

		this.stateIdMapping = StateMetadataUtils.getStateIdMapping(serializationProxy);
		final StreamCompressionDecorator streamCompressionDecorator = StateMetadataUtils
				.getCompressionDecorator(serializationProxy);

		final KeyGroupRangeOffsets rangeOffsets = keyGroupsStateHandle.getGroupRangeOffsets();
		LOGGER.debug("{}", rangeOffsets);

		offsetsIt = new ValidOffsetsIterator(rangeOffsets);

		hasNext = seekNextOffset();

		if (hasNext) {
			final InputStream compressedInputStream = streamCompressionDecorator
					.decorateWithCompression(stateHandleInStream);
			compressedInputView = new DataInputViewStreamWrapper(compressedInputStream);
			seekNextStateId(false);
		}
	}

	return hasNext;
}
Example #14
Source File: HeapRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
Example #15
Source File: ValidOffsetsIterator.java    From bravo with Apache License 2.0 4 votes vote down vote up 
public ValidOffsetsIterator(KeyGroupRangeOffsets keyGroupRangeOffsets) {
	delegate = StreamSupport.stream(keyGroupRangeOffsets.spliterator(), false)
			.map(tuple -> tuple.f1)
			.filter(offset -> offset > 0)
			.iterator();
}
Example #16
Source File: CheckpointTestUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static KeyGroupsStateHandle createDummyKeyGroupStateHandle(Random rnd) {
	return new KeyGroupsStateHandle(
		new KeyGroupRangeOffsets(1, 1, new long[]{rnd.nextInt(1024)}),
		createDummyStreamStateHandle(rnd));
}
Example #17
Source File: CheckpointTestUtils.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public static KeyGroupsStateHandle createDummyKeyGroupStateHandle(Random rnd) {
	return new KeyGroupsStateHandle(
		new KeyGroupRangeOffsets(1, 1, new long[]{rnd.nextInt(1024)}),
		createDummyStreamStateHandle(rnd));
}
Example #18
Source File: CheckpointTestUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static KeyGroupsStateHandle createDummyKeyGroupStateHandle(Random rnd, String basePath) {
	return new KeyGroupsStateHandle(
		new KeyGroupRangeOffsets(1, 1, new long[]{rnd.nextInt(1024)}),
		createDummyStreamStateHandle(rnd, basePath));
}
Example #19
Source File: CheckpointCoordinatorTestingUtils.java    From flink with Apache License 2.0 3 votes vote down vote up 
public static KeyGroupsStateHandle generateKeyGroupState(
	KeyGroupRange keyGroupRange,
	List<? extends Serializable> states) throws IOException {

	Preconditions.checkArgument(keyGroupRange.getNumberOfKeyGroups() == states.size());

	Tuple2<byte[], List<long[]>> serializedDataWithOffsets =
		serializeTogetherAndTrackOffsets(Collections.<List<? extends Serializable>>singletonList(states));

	KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, serializedDataWithOffsets.f1.get(0));

	ByteStreamStateHandle allSerializedStatesHandle = generateByteStreamStateHandle(serializedDataWithOffsets.f0);

	return new KeyGroupsStateHandle(keyGroupRangeOffsets, allSerializedStatesHandle);
}