Java Code Examples for org.apache.flink.runtime.state.KeyGroupRange#getStartKeyGroup()

/**
 * Registers a KvState instance for the given key group index.
 *
 * @param keyGroupRange  Key group range to register
 * @param kvStateId      ID of the KvState instance at the key group index.
 * @param kvStateAddress Server address of the KvState instance at the key group index.
 * @throws IndexOutOfBoundsException If key group range start < 0 or key group range end >= Number of key groups
 */
public void registerKvState(KeyGroupRange keyGroupRange, KvStateID kvStateId, InetSocketAddress kvStateAddress) {

	if (keyGroupRange.getStartKeyGroup() < 0 || keyGroupRange.getEndKeyGroup() >= numKeyGroups) {
		throw new IndexOutOfBoundsException("Key group index");
	}

	for (int kgIdx = keyGroupRange.getStartKeyGroup(); kgIdx <= keyGroupRange.getEndKeyGroup(); ++kgIdx) {

		if (kvStateIds[kgIdx] == null && kvStateAddresses[kgIdx] == null) {
			numRegisteredKeyGroups++;
		}

		kvStateIds[kgIdx] = kvStateId;
		kvStateAddresses[kgIdx] = kvStateAddress;
	}
}
 

/**
 * Registers a KvState instance for the given key group index.
 *
 * @param keyGroupRange Key group range to unregister.
 * @throws IndexOutOfBoundsException If key group range start < 0 or key group range end >= Number of key groups
 * @throws IllegalArgumentException  If no location information registered for a key group index in the range.
 */
void unregisterKvState(KeyGroupRange keyGroupRange) {
	if (keyGroupRange.getStartKeyGroup() < 0 || keyGroupRange.getEndKeyGroup() >= numKeyGroups) {
		throw new IndexOutOfBoundsException("Key group index");
	}

	for (int kgIdx = keyGroupRange.getStartKeyGroup(); kgIdx <= keyGroupRange.getEndKeyGroup(); ++kgIdx) {
		if (kvStateIds[kgIdx] == null || kvStateAddresses[kgIdx] == null) {
			throw new IllegalArgumentException("Not registered. Probably registration/unregistration race.");
		}

		numRegisteredKeyGroups--;

		kvStateIds[kgIdx] = null;
		kvStateAddresses[kgIdx] = null;
	}
}
 

@SuppressWarnings("unchecked")
public KeyGroupPartitionedPriorityQueue(
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull PartitionQueueSetFactory<T, PQ> orderedCacheFactory,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups) {

	this.keyExtractor = keyExtractor;
	this.totalKeyGroups = totalKeyGroups;
	this.firstKeyGroup = keyGroupRange.getStartKeyGroup();
	this.keyGroupedHeaps = (PQ[]) new InternalPriorityQueue[keyGroupRange.getNumberOfKeyGroups()];
	this.heapOfKeyGroupedHeaps = new HeapPriorityQueue<>(
		new InternalPriorityQueueComparator<>(elementPriorityComparator),
		keyGroupRange.getNumberOfKeyGroups());
	for (int i = 0; i < keyGroupedHeaps.length; i++) {
		final PQ keyGroupSubHeap =
			orderedCacheFactory.create(firstKeyGroup + i, totalKeyGroups, keyExtractor, elementPriorityComparator);
		keyGroupedHeaps[i] = keyGroupSubHeap;
		heapOfKeyGroupedHeaps.add(keyGroupSubHeap);
	}
}
 

/**
 * Registers a KvState instance for the given key group index.
 *
 * @param keyGroupRange  Key group range to register
 * @param kvStateId      ID of the KvState instance at the key group index.
 * @param kvStateAddress Server address of the KvState instance at the key group index.
 * @throws IndexOutOfBoundsException If key group range start < 0 or key group range end >= Number of key groups
 */
public void registerKvState(KeyGroupRange keyGroupRange, KvStateID kvStateId, InetSocketAddress kvStateAddress) {

	if (keyGroupRange.getStartKeyGroup() < 0 || keyGroupRange.getEndKeyGroup() >= numKeyGroups) {
		throw new IndexOutOfBoundsException("Key group index");
	}

	for (int kgIdx = keyGroupRange.getStartKeyGroup(); kgIdx <= keyGroupRange.getEndKeyGroup(); ++kgIdx) {

		if (kvStateIds[kgIdx] == null && kvStateAddresses[kgIdx] == null) {
			numRegisteredKeyGroups++;
		}

		kvStateIds[kgIdx] = kvStateId;
		kvStateAddresses[kgIdx] = kvStateAddress;
	}
}
 

/**
 * Registers a KvState instance for the given key group index.
 *
 * @param keyGroupRange Key group range to unregister.
 * @throws IndexOutOfBoundsException If key group range start < 0 or key group range end >= Number of key groups
 * @throws IllegalArgumentException  If no location information registered for a key group index in the range.
 */
void unregisterKvState(KeyGroupRange keyGroupRange) {
	if (keyGroupRange.getStartKeyGroup() < 0 || keyGroupRange.getEndKeyGroup() >= numKeyGroups) {
		throw new IndexOutOfBoundsException("Key group index");
	}

	for (int kgIdx = keyGroupRange.getStartKeyGroup(); kgIdx <= keyGroupRange.getEndKeyGroup(); ++kgIdx) {
		if (kvStateIds[kgIdx] == null || kvStateAddresses[kgIdx] == null) {
			throw new IllegalArgumentException("Not registered. Probably registration/unregistration race.");
		}

		numRegisteredKeyGroups--;

		kvStateIds[kgIdx] = null;
		kvStateAddresses[kgIdx] = null;
	}
}
 

@SuppressWarnings("unchecked")
public KeyGroupPartitionedPriorityQueue(
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull PartitionQueueSetFactory<T, PQ> orderedCacheFactory,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups) {

	this.keyExtractor = keyExtractor;
	this.totalKeyGroups = totalKeyGroups;
	this.firstKeyGroup = keyGroupRange.getStartKeyGroup();
	this.keyGroupedHeaps = (PQ[]) new InternalPriorityQueue[keyGroupRange.getNumberOfKeyGroups()];
	this.heapOfKeyGroupedHeaps = new HeapPriorityQueue<>(
		new InternalPriorityQueueComparator<>(elementPriorityComparator),
		keyGroupRange.getNumberOfKeyGroups());
	for (int i = 0; i < keyGroupedHeaps.length; i++) {
		final PQ keyGroupSubHeap =
			orderedCacheFactory.create(firstKeyGroup + i, totalKeyGroups, keyExtractor, elementPriorityComparator);
		keyGroupedHeaps[i] = keyGroupSubHeap;
		heapOfKeyGroupedHeaps.add(keyGroupSubHeap);
	}
}
 

/**
 * Registers a KvState instance for the given key group index.
 *
 * @param keyGroupRange  Key group range to register
 * @param kvStateId      ID of the KvState instance at the key group index.
 * @param kvStateAddress Server address of the KvState instance at the key group index.
 * @throws IndexOutOfBoundsException If key group range start < 0 or key group range end >= Number of key groups
 */
public void registerKvState(KeyGroupRange keyGroupRange, KvStateID kvStateId, InetSocketAddress kvStateAddress) {

	if (keyGroupRange.getStartKeyGroup() < 0 || keyGroupRange.getEndKeyGroup() >= numKeyGroups) {
		throw new IndexOutOfBoundsException("Key group index");
	}

	for (int kgIdx = keyGroupRange.getStartKeyGroup(); kgIdx <= keyGroupRange.getEndKeyGroup(); ++kgIdx) {

		if (kvStateIds[kgIdx] == null && kvStateAddresses[kgIdx] == null) {
			numRegisteredKeyGroups++;
		}

		kvStateIds[kgIdx] = kvStateId;
		kvStateAddresses[kgIdx] = kvStateAddress;
	}
}
 

/**
 * Registers a KvState instance for the given key group index.
 *
 * @param keyGroupRange Key group range to unregister.
 * @throws IndexOutOfBoundsException If key group range start < 0 or key group range end >= Number of key groups
 * @throws IllegalArgumentException  If no location information registered for a key group index in the range.
 */
void unregisterKvState(KeyGroupRange keyGroupRange) {
	if (keyGroupRange.getStartKeyGroup() < 0 || keyGroupRange.getEndKeyGroup() >= numKeyGroups) {
		throw new IndexOutOfBoundsException("Key group index");
	}

	for (int kgIdx = keyGroupRange.getStartKeyGroup(); kgIdx <= keyGroupRange.getEndKeyGroup(); ++kgIdx) {
		if (kvStateIds[kgIdx] == null || kvStateAddresses[kgIdx] == null) {
			throw new IllegalArgumentException("Not registered. Probably registration/unregistration race.");
		}

		numRegisteredKeyGroups--;

		kvStateIds[kgIdx] = null;
		kvStateAddresses[kgIdx] = null;
	}
}
 

@SuppressWarnings("unchecked")
public KeyGroupPartitionedPriorityQueue(
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull PartitionQueueSetFactory<T, PQ> orderedCacheFactory,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups) {

	this.keyExtractor = keyExtractor;
	this.totalKeyGroups = totalKeyGroups;
	this.firstKeyGroup = keyGroupRange.getStartKeyGroup();
	this.keyGroupedHeaps = (PQ[]) new InternalPriorityQueue[keyGroupRange.getNumberOfKeyGroups()];
	this.heapOfKeyGroupedHeaps = new HeapPriorityQueue<>(
		new InternalPriorityQueueComparator<>(elementPriorityComparator),
		keyGroupRange.getNumberOfKeyGroups());
	for (int i = 0; i < keyGroupedHeaps.length; i++) {
		final PQ keyGroupSubHeap =
			orderedCacheFactory.create(firstKeyGroup + i, totalKeyGroups, keyExtractor, elementPriorityComparator);
		keyGroupedHeaps[i] = keyGroupSubHeap;
		heapOfKeyGroupedHeaps.add(keyGroupSubHeap);
	}
}
 

/**
 * The method to clip the db instance according to the target key group range using
 * the {@link RocksDB#delete(ColumnFamilyHandle, byte[])}.
 *
 * @param db the RocksDB instance to be clipped.
 * @param columnFamilyHandles the column families in the db instance.
 * @param targetKeyGroupRange the target key group range.
 * @param currentKeyGroupRange the key group range of the db instance.
 * @param keyGroupPrefixBytes Number of bytes required to prefix the key groups.
 */
public static void clipDBWithKeyGroupRange(
	@Nonnull RocksDB db,
	@Nonnull List<ColumnFamilyHandle> columnFamilyHandles,
	@Nonnull KeyGroupRange targetKeyGroupRange,
	@Nonnull KeyGroupRange currentKeyGroupRange,
	@Nonnegative int keyGroupPrefixBytes) throws RocksDBException {

	final byte[] beginKeyGroupBytes = new byte[keyGroupPrefixBytes];
	final byte[] endKeyGroupBytes = new byte[keyGroupPrefixBytes];

	if (currentKeyGroupRange.getStartKeyGroup() < targetKeyGroupRange.getStartKeyGroup()) {
		RocksDBKeySerializationUtils.serializeKeyGroup(
			currentKeyGroupRange.getStartKeyGroup(), beginKeyGroupBytes);
		RocksDBKeySerializationUtils.serializeKeyGroup(
			targetKeyGroupRange.getStartKeyGroup(), endKeyGroupBytes);
		deleteRange(db, columnFamilyHandles, beginKeyGroupBytes, endKeyGroupBytes);
	}

	if (currentKeyGroupRange.getEndKeyGroup() > targetKeyGroupRange.getEndKeyGroup()) {
		RocksDBKeySerializationUtils.serializeKeyGroup(
			targetKeyGroupRange.getEndKeyGroup() + 1, beginKeyGroupBytes);
		RocksDBKeySerializationUtils.serializeKeyGroup(
			currentKeyGroupRange.getEndKeyGroup() + 1, endKeyGroupBytes);
		deleteRange(db, columnFamilyHandles, beginKeyGroupBytes, endKeyGroupBytes);
	}
}
 

/**
 * Creates deep copy of the given {@link KeyedStateHandle}.
 */
public static KeyedStateHandle deepDummyCopy(KeyedStateHandle original) {

	if (original == null) {
		return null;
	}

	KeyGroupRange keyGroupRange = original.getKeyGroupRange();
	return new DummyKeyedStateHandle(
		new KeyGroupRange(keyGroupRange.getStartKeyGroup(), keyGroupRange.getEndKeyGroup()));
}
 

/**
 * The method to clip the db instance according to the target key group range using
 * the {@link RocksDB#delete(ColumnFamilyHandle, byte[])}.
 *
 * @param db the RocksDB instance to be clipped.
 * @param columnFamilyHandles the column families in the db instance.
 * @param targetKeyGroupRange the target key group range.
 * @param currentKeyGroupRange the key group range of the db instance.
 * @param keyGroupPrefixBytes Number of bytes required to prefix the key groups.
 */
public static void clipDBWithKeyGroupRange(
	@Nonnull RocksDB db,
	@Nonnull List<ColumnFamilyHandle> columnFamilyHandles,
	@Nonnull KeyGroupRange targetKeyGroupRange,
	@Nonnull KeyGroupRange currentKeyGroupRange,
	@Nonnegative int keyGroupPrefixBytes) throws RocksDBException {

	final byte[] beginKeyGroupBytes = new byte[keyGroupPrefixBytes];
	final byte[] endKeyGroupBytes = new byte[keyGroupPrefixBytes];

	if (currentKeyGroupRange.getStartKeyGroup() < targetKeyGroupRange.getStartKeyGroup()) {
		RocksDBKeySerializationUtils.serializeKeyGroup(
			currentKeyGroupRange.getStartKeyGroup(), beginKeyGroupBytes);
		RocksDBKeySerializationUtils.serializeKeyGroup(
			targetKeyGroupRange.getStartKeyGroup(), endKeyGroupBytes);
		deleteRange(db, columnFamilyHandles, beginKeyGroupBytes, endKeyGroupBytes);
	}

	if (currentKeyGroupRange.getEndKeyGroup() > targetKeyGroupRange.getEndKeyGroup()) {
		RocksDBKeySerializationUtils.serializeKeyGroup(
			targetKeyGroupRange.getEndKeyGroup() + 1, beginKeyGroupBytes);
		RocksDBKeySerializationUtils.serializeKeyGroup(
			currentKeyGroupRange.getEndKeyGroup() + 1, endKeyGroupBytes);
		deleteRange(db, columnFamilyHandles, beginKeyGroupBytes, endKeyGroupBytes);
	}
}
 

/**
 * Creates deep copy of the given {@link KeyedStateHandle}.
 */
public static KeyedStateHandle deepDummyCopy(KeyedStateHandle original) {

	if (original == null) {
		return null;
	}

	KeyGroupRange keyGroupRange = original.getKeyGroupRange();
	return new DummyKeyedStateHandle(
		new KeyGroupRange(keyGroupRange.getStartKeyGroup(), keyGroupRange.getEndKeyGroup()));
}
 

/**
 * The method to clip the db instance according to the target key group range using
 * the {@link RocksDB#delete(ColumnFamilyHandle, byte[])}.
 *
 * @param db the RocksDB instance to be clipped.
 * @param columnFamilyHandles the column families in the db instance.
 * @param targetKeyGroupRange the target key group range.
 * @param currentKeyGroupRange the key group range of the db instance.
 * @param keyGroupPrefixBytes Number of bytes required to prefix the key groups.
 */
public static void clipDBWithKeyGroupRange(
	@Nonnull RocksDB db,
	@Nonnull List<ColumnFamilyHandle> columnFamilyHandles,
	@Nonnull KeyGroupRange targetKeyGroupRange,
	@Nonnull KeyGroupRange currentKeyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnegative long writeBatchSize) throws RocksDBException {

	final byte[] beginKeyGroupBytes = new byte[keyGroupPrefixBytes];
	final byte[] endKeyGroupBytes = new byte[keyGroupPrefixBytes];

	if (currentKeyGroupRange.getStartKeyGroup() < targetKeyGroupRange.getStartKeyGroup()) {
		RocksDBKeySerializationUtils.serializeKeyGroup(
			currentKeyGroupRange.getStartKeyGroup(), beginKeyGroupBytes);
		RocksDBKeySerializationUtils.serializeKeyGroup(
			targetKeyGroupRange.getStartKeyGroup(), endKeyGroupBytes);
		deleteRange(db, columnFamilyHandles, beginKeyGroupBytes, endKeyGroupBytes, writeBatchSize);
	}

	if (currentKeyGroupRange.getEndKeyGroup() > targetKeyGroupRange.getEndKeyGroup()) {
		RocksDBKeySerializationUtils.serializeKeyGroup(
			targetKeyGroupRange.getEndKeyGroup() + 1, beginKeyGroupBytes);
		RocksDBKeySerializationUtils.serializeKeyGroup(
			currentKeyGroupRange.getEndKeyGroup() + 1, endKeyGroupBytes);
		deleteRange(db, columnFamilyHandles, beginKeyGroupBytes, endKeyGroupBytes, writeBatchSize);
	}
}
 

/**
 * Creates deep copy of the given {@link KeyedStateHandle}.
 */
public static KeyedStateHandle deepDummyCopy(KeyedStateHandle original) {

	if (original == null) {
		return null;
	}

	KeyGroupRange keyGroupRange = original.getKeyGroupRange();
	return new DummyKeyedStateHandle(
		new KeyGroupRange(keyGroupRange.getStartKeyGroup(), keyGroupRange.getEndKeyGroup()));
}