org.apache.flink.runtime.state.PriorityComparator Java Examples

/**
 * Creates an empty {@link HeapPriorityQueueSet} with the requested initial capacity.
 *
 * @param elementPriorityComparator comparator for the priority of contained elements.
 * @param keyExtractor function to extract a key from the contained elements.
 * @param minimumCapacity the minimum and initial capacity of this priority queue.
 * @param keyGroupRange the key-group range of the elements in this set.
 * @param totalNumberOfKeyGroups the total number of key-groups of the job.
 */
@SuppressWarnings("unchecked")
public HeapPriorityQueueSet(
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnegative int minimumCapacity,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalNumberOfKeyGroups) {

	super(elementPriorityComparator, minimumCapacity);

	this.keyExtractor = keyExtractor;

	this.totalNumberOfKeyGroups = totalNumberOfKeyGroups;
	this.keyGroupRange = keyGroupRange;

	final int keyGroupsInLocalRange = keyGroupRange.getNumberOfKeyGroups();
	final int deduplicationSetSize = 1 + minimumCapacity / keyGroupsInLocalRange;
	this.deduplicationMapsByKeyGroup = new HashMap[keyGroupsInLocalRange];
	for (int i = 0; i < keyGroupsInLocalRange; ++i) {
		deduplicationMapsByKeyGroup[i] = new HashMap<>(deduplicationSetSize);
	}
}
 

@SuppressWarnings("unchecked")
public KeyGroupPartitionedPriorityQueue(
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull PartitionQueueSetFactory<T, PQ> orderedCacheFactory,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups) {

	this.keyExtractor = keyExtractor;
	this.totalKeyGroups = totalKeyGroups;
	this.firstKeyGroup = keyGroupRange.getStartKeyGroup();
	this.keyGroupedHeaps = (PQ[]) new InternalPriorityQueue[keyGroupRange.getNumberOfKeyGroups()];
	this.heapOfKeyGroupedHeaps = new HeapPriorityQueue<>(
		new InternalPriorityQueueComparator<>(elementPriorityComparator),
		keyGroupRange.getNumberOfKeyGroups());
	for (int i = 0; i < keyGroupedHeaps.length; i++) {
		final PQ keyGroupSubHeap =
			orderedCacheFactory.create(firstKeyGroup + i, totalKeyGroups, keyExtractor, elementPriorityComparator);
		keyGroupedHeaps[i] = keyGroupSubHeap;
		heapOfKeyGroupedHeaps.add(keyGroupSubHeap);
	}
}
 

@SuppressWarnings("unchecked")
public KeyGroupPartitionedPriorityQueue(
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull PartitionQueueSetFactory<T, PQ> orderedCacheFactory,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups) {

	this.keyExtractor = keyExtractor;
	this.totalKeyGroups = totalKeyGroups;
	this.firstKeyGroup = keyGroupRange.getStartKeyGroup();
	this.keyGroupedHeaps = (PQ[]) new InternalPriorityQueue[keyGroupRange.getNumberOfKeyGroups()];
	this.heapOfKeyGroupedHeaps = new HeapPriorityQueue<>(
		new InternalPriorityQueueComparator<>(elementPriorityComparator),
		keyGroupRange.getNumberOfKeyGroups());
	for (int i = 0; i < keyGroupedHeaps.length; i++) {
		final PQ keyGroupSubHeap =
			orderedCacheFactory.create(firstKeyGroup + i, totalKeyGroups, keyExtractor, elementPriorityComparator);
		keyGroupedHeaps[i] = keyGroupSubHeap;
		heapOfKeyGroupedHeaps.add(keyGroupSubHeap);
	}
}
 

/**
 * Creates an empty {@link HeapPriorityQueueSet} with the requested initial capacity.
 *
 * @param elementPriorityComparator comparator for the priority of contained elements.
 * @param keyExtractor function to extract a key from the contained elements.
 * @param minimumCapacity the minimum and initial capacity of this priority queue.
 * @param keyGroupRange the key-group range of the elements in this set.
 * @param totalNumberOfKeyGroups the total number of key-groups of the job.
 */
@SuppressWarnings("unchecked")
public HeapPriorityQueueSet(
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnegative int minimumCapacity,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalNumberOfKeyGroups) {

	super(elementPriorityComparator, minimumCapacity);

	this.keyExtractor = keyExtractor;

	this.totalNumberOfKeyGroups = totalNumberOfKeyGroups;
	this.keyGroupRange = keyGroupRange;

	final int keyGroupsInLocalRange = keyGroupRange.getNumberOfKeyGroups();
	final int deduplicationSetSize = 1 + minimumCapacity / keyGroupsInLocalRange;
	this.deduplicationMapsByKeyGroup = new HashMap[keyGroupsInLocalRange];
	for (int i = 0; i < keyGroupsInLocalRange; ++i) {
		deduplicationMapsByKeyGroup[i] = new HashMap<>(deduplicationSetSize);
	}
}
 

/**
 * Creates an empty {@link HeapPriorityQueueSet} with the requested initial capacity.
 *
 * @param elementPriorityComparator comparator for the priority of contained elements.
 * @param keyExtractor function to extract a key from the contained elements.
 * @param minimumCapacity the minimum and initial capacity of this priority queue.
 * @param keyGroupRange the key-group range of the elements in this set.
 * @param totalNumberOfKeyGroups the total number of key-groups of the job.
 */
@SuppressWarnings("unchecked")
public HeapPriorityQueueSet(
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnegative int minimumCapacity,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalNumberOfKeyGroups) {

	super(elementPriorityComparator, minimumCapacity);

	this.keyExtractor = keyExtractor;

	this.totalNumberOfKeyGroups = totalNumberOfKeyGroups;
	this.keyGroupRange = keyGroupRange;

	final int keyGroupsInLocalRange = keyGroupRange.getNumberOfKeyGroups();
	final int deduplicationSetSize = 1 + minimumCapacity / keyGroupsInLocalRange;
	this.deduplicationMapsByKeyGroup = new HashMap[keyGroupsInLocalRange];
	for (int i = 0; i < keyGroupsInLocalRange; ++i) {
		deduplicationMapsByKeyGroup[i] = new HashMap<>(deduplicationSetSize);
	}
}
 

@SuppressWarnings("unchecked")
public KeyGroupPartitionedPriorityQueue(
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull PartitionQueueSetFactory<T, PQ> orderedCacheFactory,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups) {

	this.keyExtractor = keyExtractor;
	this.totalKeyGroups = totalKeyGroups;
	this.firstKeyGroup = keyGroupRange.getStartKeyGroup();
	this.keyGroupedHeaps = (PQ[]) new InternalPriorityQueue[keyGroupRange.getNumberOfKeyGroups()];
	this.heapOfKeyGroupedHeaps = new HeapPriorityQueue<>(
		new InternalPriorityQueueComparator<>(elementPriorityComparator),
		keyGroupRange.getNumberOfKeyGroups());
	for (int i = 0; i < keyGroupedHeaps.length; i++) {
		final PQ keyGroupSubHeap =
			orderedCacheFactory.create(firstKeyGroup + i, totalKeyGroups, keyExtractor, elementPriorityComparator);
		keyGroupedHeaps[i] = keyGroupSubHeap;
		heapOfKeyGroupedHeaps.add(keyGroupSubHeap);
	}
}
 

@Nonnull
@Override
public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> HeapPriorityQueueSet<T> create(
	@Nonnull String stateName,
	@Nonnull TypeSerializer<T> byteOrderedElementSerializer) {

	return new HeapPriorityQueueSet<>(
		PriorityComparator.forPriorityComparableObjects(),
		KeyExtractorFunction.forKeyedObjects(),
		minimumCapacity,
		keyGroupRange,
		totalKeyGroups);
}
 

@Nonnull
@Override
public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T>
create(
	@Nonnull String stateName,
	@Nonnull TypeSerializer<T> byteOrderedElementSerializer) {
	return new HeapPriorityQueueSet<>(
		PriorityComparator.forPriorityComparableObjects(),
		KeyExtractorFunction.forKeyedObjects(),
		0,
		keyGroupRange,
		0);
}
 

public KeyGroupHeapPQSet(
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	int minimumCapacity,
	@Nonnull KeyGroupRange keyGroupRange,
	int totalNumberOfKeyGroups) {
	super(elementPriorityComparator, keyExtractor, minimumCapacity, keyGroupRange, totalNumberOfKeyGroups);
	this.internalIndex = HeapPriorityQueueElement.NOT_CONTAINED;
}
 

/**
 * Creates an empty {@link HeapPriorityQueue} with the requested initial capacity.
 *
 * @param elementPriorityComparator comparator for the priority of contained elements.
 * @param minimumCapacity the minimum and initial capacity of this priority queue.
 */
@SuppressWarnings("unchecked")
public HeapPriorityQueue(
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnegative int minimumCapacity) {
	super(minimumCapacity);
	this.elementPriorityComparator = elementPriorityComparator;
}
 

@Nonnull
@Override
public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> HeapPriorityQueueSet<T> create(
	@Nonnull String stateName,
	@Nonnull TypeSerializer<T> byteOrderedElementSerializer) {

	return new HeapPriorityQueueSet<>(
		PriorityComparator.forPriorityComparableObjects(),
		KeyExtractorFunction.forKeyedObjects(),
		minimumCapacity,
		keyGroupRange,
		totalKeyGroups);
}
 

@Nonnull
@Override
public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T>
create(@Nonnull String stateName, @Nonnull TypeSerializer<T> byteOrderedElementSerializer) {

	final RocksDBKeyedStateBackend.RocksDbKvStateInfo stateCFHandle =
		tryRegisterPriorityQueueMetaInfo(stateName, byteOrderedElementSerializer);

	final ColumnFamilyHandle columnFamilyHandle = stateCFHandle.columnFamilyHandle;

	return new KeyGroupPartitionedPriorityQueue<>(
		KeyExtractorFunction.forKeyedObjects(),
		PriorityComparator.forPriorityComparableObjects(),
		new KeyGroupPartitionedPriorityQueue.PartitionQueueSetFactory<T, RocksDBCachingPriorityQueueSet<T>>() {
			@Nonnull
			@Override
			public RocksDBCachingPriorityQueueSet<T> create(
				int keyGroupId,
				int numKeyGroups,
				@Nonnull KeyExtractorFunction<T> keyExtractor,
				@Nonnull PriorityComparator<T> elementPriorityComparator) {
				TreeOrderedSetCache orderedSetCache = new TreeOrderedSetCache(DEFAULT_CACHES_SIZE);
				return new RocksDBCachingPriorityQueueSet<>(
					keyGroupId,
					keyGroupPrefixBytes,
					db,
					readOptions,
					columnFamilyHandle,
					byteOrderedElementSerializer,
					sharedElementOutView,
					sharedElementInView,
					writeBatchWrapper,
					orderedSetCache
				);
			}
		},
		keyGroupRange,
		numberOfKeyGroups);
}
 

@Nonnull
@Override
public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T>
create(
	@Nonnull String stateName,
	@Nonnull TypeSerializer<T> byteOrderedElementSerializer) {
	return new HeapPriorityQueueSet<>(
		PriorityComparator.forPriorityComparableObjects(),
		KeyExtractorFunction.forKeyedObjects(),
		0,
		keyGroupRange,
		0);
}
 

public KeyGroupHeapPQSet(
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	int minimumCapacity,
	@Nonnull KeyGroupRange keyGroupRange,
	int totalNumberOfKeyGroups) {
	super(elementPriorityComparator, keyExtractor, minimumCapacity, keyGroupRange, totalNumberOfKeyGroups);
	this.internalIndex = HeapPriorityQueueElement.NOT_CONTAINED;
}
 

/**
 * Creates an empty {@link HeapPriorityQueue} with the requested initial capacity.
 *
 * @param elementPriorityComparator comparator for the priority of contained elements.
 * @param minimumCapacity the minimum and initial capacity of this priority queue.
 */
@SuppressWarnings("unchecked")
public HeapPriorityQueue(
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnegative int minimumCapacity) {
	super(minimumCapacity);
	this.elementPriorityComparator = elementPriorityComparator;
}
 

@Nonnull
@Override
public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> HeapPriorityQueueSet<T> create(
	@Nonnull String stateName,
	@Nonnull TypeSerializer<T> byteOrderedElementSerializer) {

	return new HeapPriorityQueueSet<>(
		PriorityComparator.forPriorityComparableObjects(),
		KeyExtractorFunction.forKeyedObjects(),
		minimumCapacity,
		keyGroupRange,
		totalKeyGroups);
}
 

@Nonnull
@Override
public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T>
create(@Nonnull String stateName, @Nonnull TypeSerializer<T> byteOrderedElementSerializer) {

	final RocksDBKeyedStateBackend.RocksDbKvStateInfo stateCFHandle =
		tryRegisterPriorityQueueMetaInfo(stateName, byteOrderedElementSerializer);

	final ColumnFamilyHandle columnFamilyHandle = stateCFHandle.columnFamilyHandle;

	return new KeyGroupPartitionedPriorityQueue<>(
		KeyExtractorFunction.forKeyedObjects(),
		PriorityComparator.forPriorityComparableObjects(),
		new KeyGroupPartitionedPriorityQueue.PartitionQueueSetFactory<T, RocksDBCachingPriorityQueueSet<T>>() {
			@Nonnull
			@Override
			public RocksDBCachingPriorityQueueSet<T> create(
				int keyGroupId,
				int numKeyGroups,
				@Nonnull KeyExtractorFunction<T> keyExtractor,
				@Nonnull PriorityComparator<T> elementPriorityComparator) {
				TreeOrderedSetCache orderedSetCache = new TreeOrderedSetCache(DEFAULT_CACHES_SIZE);
				return new RocksDBCachingPriorityQueueSet<>(
					keyGroupId,
					keyGroupPrefixBytes,
					db,
					columnFamilyHandle,
					byteOrderedElementSerializer,
					sharedElementOutView,
					sharedElementInView,
					writeBatchWrapper,
					orderedSetCache
				);
			}
		},
		keyGroupRange,
		numberOfKeyGroups);
}
 

@Nonnull
@Override
public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T>
create(
	@Nonnull String stateName,
	@Nonnull TypeSerializer<T> byteOrderedElementSerializer) {
	return new HeapPriorityQueueSet<>(
		PriorityComparator.forPriorityComparableObjects(),
		KeyExtractorFunction.forKeyedObjects(),
		0,
		keyGroupRange,
		0);
}
 

public KeyGroupHeapPQSet(
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	int minimumCapacity,
	@Nonnull KeyGroupRange keyGroupRange,
	int totalNumberOfKeyGroups) {
	super(elementPriorityComparator, keyExtractor, minimumCapacity, keyGroupRange, totalNumberOfKeyGroups);
	this.internalIndex = HeapPriorityQueueElement.NOT_CONTAINED;
}
 

/**
 * Creates an empty {@link HeapPriorityQueue} with the requested initial capacity.
 *
 * @param elementPriorityComparator comparator for the priority of contained elements.
 * @param minimumCapacity the minimum and initial capacity of this priority queue.
 */
@SuppressWarnings("unchecked")
public HeapPriorityQueue(
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnegative int minimumCapacity) {
	super(minimumCapacity);
	this.elementPriorityComparator = elementPriorityComparator;
}
 

@Nonnull
@Override
public <T extends HeapPriorityQueueElement & PriorityComparable & Keyed> KeyGroupedInternalPriorityQueue<T>
create(@Nonnull String stateName, @Nonnull TypeSerializer<T> byteOrderedElementSerializer) {

	final RocksDBKeyedStateBackend.RocksDbKvStateInfo stateCFHandle =
		tryRegisterPriorityQueueMetaInfo(stateName, byteOrderedElementSerializer);

	final ColumnFamilyHandle columnFamilyHandle = stateCFHandle.columnFamilyHandle;

	return new KeyGroupPartitionedPriorityQueue<>(
		KeyExtractorFunction.forKeyedObjects(),
		PriorityComparator.forPriorityComparableObjects(),
		new KeyGroupPartitionedPriorityQueue.PartitionQueueSetFactory<T, RocksDBCachingPriorityQueueSet<T>>() {
			@Nonnull
			@Override
			public RocksDBCachingPriorityQueueSet<T> create(
				int keyGroupId,
				int numKeyGroups,
				@Nonnull KeyExtractorFunction<T> keyExtractor,
				@Nonnull PriorityComparator<T> elementPriorityComparator) {
				TreeOrderedSetCache orderedSetCache = new TreeOrderedSetCache(DEFAULT_CACHES_SIZE);
				return new RocksDBCachingPriorityQueueSet<>(
					keyGroupId,
					keyGroupPrefixBytes,
					db,
					columnFamilyHandle,
					byteOrderedElementSerializer,
					sharedElementOutView,
					sharedElementInView,
					writeBatchWrapper,
					orderedSetCache
				);
			}
		},
		keyGroupRange,
		numberOfKeyGroups);
}
 

InternalPriorityQueueComparator(@Nonnull PriorityComparator<T> elementPriorityComparator) {
	this.elementPriorityComparator = elementPriorityComparator;
}
 

InternalPriorityQueueComparator(@Nonnull PriorityComparator<T> elementPriorityComparator) {
	this.elementPriorityComparator = elementPriorityComparator;
}
 

InternalPriorityQueueComparator(@Nonnull PriorityComparator<T> elementPriorityComparator) {
	this.elementPriorityComparator = elementPriorityComparator;
}
 

/**
 * Creates a new queue for a given key-group partition.
 *
 * @param keyGroupId the key-group of the elements managed by the produced queue.
 * @param numKeyGroups the total number of key-groups in the job.
 * @param elementPriorityComparator the comparator that determines the order of managed elements by priority.
 * @return a new queue for the given key-group.
 */
@Nonnull
PQS create(
	@Nonnegative int keyGroupId,
	@Nonnegative int numKeyGroups,
	@Nonnull KeyExtractorFunction<T> keyExtractorFunction,
	@Nonnull PriorityComparator<T> elementPriorityComparator);
 

/**
 * Creates a new queue for a given key-group partition.
 *
 * @param keyGroupId the key-group of the elements managed by the produced queue.
 * @param numKeyGroups the total number of key-groups in the job.
 * @param elementPriorityComparator the comparator that determines the order of managed elements by priority.
 * @return a new queue for the given key-group.
 */
@Nonnull
PQS create(
	@Nonnegative int keyGroupId,
	@Nonnegative int numKeyGroups,
	@Nonnull KeyExtractorFunction<T> keyExtractorFunction,
	@Nonnull PriorityComparator<T> elementPriorityComparator);
 

/**
 * Creates a new queue for a given key-group partition.
 *
 * @param keyGroupId the key-group of the elements managed by the produced queue.
 * @param numKeyGroups the total number of key-groups in the job.
 * @param elementPriorityComparator the comparator that determines the order of managed elements by priority.
 * @return a new queue for the given key-group.
 */
@Nonnull
PQS create(
	@Nonnegative int keyGroupId,
	@Nonnegative int numKeyGroups,
	@Nonnull KeyExtractorFunction<T> keyExtractorFunction,
	@Nonnull PriorityComparator<T> elementPriorityComparator);