javax.annotation.Nonnegative Java Examples

The following examples show how to use javax.annotation.Nonnegative. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TaskLocalStateStoreImpl.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
TaskLocalStateStoreImpl(
	@Nonnull JobID jobID,
	@Nonnull AllocationID allocationID,
	@Nonnull JobVertexID jobVertexID,
	@Nonnegative int subtaskIndex,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull Executor discardExecutor,
	@Nonnull SortedMap<Long, TaskStateSnapshot> storedTaskStateByCheckpointID,
	@Nonnull Object lock) {

	this.jobID = jobID;
	this.allocationID = allocationID;
	this.jobVertexID = jobVertexID;
	this.subtaskIndex = subtaskIndex;
	this.discardExecutor = discardExecutor;
	this.localRecoveryConfig = localRecoveryConfig;
	this.storedTaskStateByCheckpointID = storedTaskStateByCheckpointID;
	this.lock = lock;
	this.disposed = false;
}
 
Example #2
Source File: RocksDBMapState.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
RocksDBMapEntry(
		@Nonnull final RocksDB db,
		@Nonnegative final int userKeyOffset,
		@Nonnull final byte[] rawKeyBytes,
		@Nonnull final byte[] rawValueBytes,
		@Nonnull final TypeSerializer<UK> keySerializer,
		@Nonnull final TypeSerializer<UV> valueSerializer,
		@Nonnull DataInputDeserializer dataInputView) {
	this.db = db;

	this.userKeyOffset = userKeyOffset;
	this.keySerializer = keySerializer;
	this.valueSerializer = valueSerializer;

	this.rawKeyBytes = rawKeyBytes;
	this.rawValueBytes = rawValueBytes;
	this.deleted = false;
	this.dataInputView = dataInputView;
}
 
Example #3
Source File: RocksDBCachingPriorityQueueSet.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
RocksDBCachingPriorityQueueSet(
	@Nonnegative int keyGroupId,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull RocksDB db,
	@Nonnull ColumnFamilyHandle columnFamilyHandle,
	@Nonnull TypeSerializer<E> byteOrderProducingSerializer,
	@Nonnull DataOutputSerializer outputStream,
	@Nonnull DataInputDeserializer inputStream,
	@Nonnull RocksDBWriteBatchWrapper batchWrapper,
	@Nonnull OrderedByteArraySetCache orderedByteArraySetCache) {
	this.db = db;
	this.columnFamilyHandle = columnFamilyHandle;
	this.byteOrderProducingSerializer = byteOrderProducingSerializer;
	this.batchWrapper = batchWrapper;
	this.outputView = outputStream;
	this.inputView = inputStream;
	this.orderedCache = orderedByteArraySetCache;
	this.allElementsInCache = false;
	this.groupPrefixBytes = createKeyGroupBytes(keyGroupId, keyGroupPrefixBytes);
	this.seekHint = groupPrefixBytes;
	this.internalIndex = HeapPriorityQueueElement.NOT_CONTAINED;
}
 
Example #4
Source File: TaskLocalStateStoreImpl.java    From flink with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
TaskLocalStateStoreImpl(
	@Nonnull JobID jobID,
	@Nonnull AllocationID allocationID,
	@Nonnull JobVertexID jobVertexID,
	@Nonnegative int subtaskIndex,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull Executor discardExecutor,
	@Nonnull SortedMap<Long, TaskStateSnapshot> storedTaskStateByCheckpointID,
	@Nonnull Object lock) {

	this.jobID = jobID;
	this.allocationID = allocationID;
	this.jobVertexID = jobVertexID;
	this.subtaskIndex = subtaskIndex;
	this.discardExecutor = discardExecutor;
	this.localRecoveryConfig = localRecoveryConfig;
	this.storedTaskStateByCheckpointID = storedTaskStateByCheckpointID;
	this.lock = lock;
	this.disposed = false;
}
 
Example #5
Source File: StateTableByKeyGroupReaders.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void readMappingsInKeyGroup(@Nonnull DataInputView inView, @Nonnegative int keyGroupId) throws IOException {

	if (inView.readByte() == 0) {
		return;
	}

	final TypeSerializer<N> namespaceSerializer = stateTable.getNamespaceSerializer();
	final TypeSerializer<S> stateSerializer = stateTable.getStateSerializer();

	// V1 uses kind of namespace compressing format
	int numNamespaces = inView.readInt();
	for (int k = 0; k < numNamespaces; k++) {
		N namespace = namespaceSerializer.deserialize(inView);
		int numEntries = inView.readInt();
		for (int l = 0; l < numEntries; l++) {
			K key = keySerializer.deserialize(inView);
			S state = stateSerializer.deserialize(inView);
			stateTable.put(key, keyGroupId, namespace, state);
		}
	}
}
 
Example #6
Source File: LocalRecoveryDirectoryProviderImpl.java    From flink with Apache License 2.0 6 votes vote down vote up
public LocalRecoveryDirectoryProviderImpl(
	@Nonnull File[] allocationBaseDirs,
	@Nonnull JobID jobID,
	@Nonnull JobVertexID jobVertexID,
	@Nonnegative int subtaskIndex) {

	Preconditions.checkArgument(allocationBaseDirs.length > 0);
	this.allocationBaseDirs = allocationBaseDirs;
	this.jobID = jobID;
	this.jobVertexID = jobVertexID;
	this.subtaskIndex = subtaskIndex;

	for (File allocationBaseDir : allocationBaseDirs) {
		Preconditions.checkNotNull(allocationBaseDir);
		allocationBaseDir.mkdirs();
	}
}
 
Example #7
Source File: KeyGroupPartitionedPriorityQueue.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public KeyGroupPartitionedPriorityQueue(
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull PartitionQueueSetFactory<T, PQ> orderedCacheFactory,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups) {

	this.keyExtractor = keyExtractor;
	this.totalKeyGroups = totalKeyGroups;
	this.firstKeyGroup = keyGroupRange.getStartKeyGroup();
	this.keyGroupedHeaps = (PQ[]) new InternalPriorityQueue[keyGroupRange.getNumberOfKeyGroups()];
	this.heapOfKeyGroupedHeaps = new HeapPriorityQueue<>(
		new InternalPriorityQueueComparator<>(elementPriorityComparator),
		keyGroupRange.getNumberOfKeyGroups());
	for (int i = 0; i < keyGroupedHeaps.length; i++) {
		final PQ keyGroupSubHeap =
			orderedCacheFactory.create(firstKeyGroup + i, totalKeyGroups, keyExtractor, elementPriorityComparator);
		keyGroupedHeaps[i] = keyGroupSubHeap;
		heapOfKeyGroupedHeaps.add(keyGroupSubHeap);
	}
}
 
Example #8
Source File: HeapPriorityQueueSet.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates an empty {@link HeapPriorityQueueSet} with the requested initial capacity.
 *
 * @param elementPriorityComparator comparator for the priority of contained elements.
 * @param keyExtractor function to extract a key from the contained elements.
 * @param minimumCapacity the minimum and initial capacity of this priority queue.
 * @param keyGroupRange the key-group range of the elements in this set.
 * @param totalNumberOfKeyGroups the total number of key-groups of the job.
 */
@SuppressWarnings("unchecked")
public HeapPriorityQueueSet(
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnegative int minimumCapacity,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalNumberOfKeyGroups) {

	super(elementPriorityComparator, minimumCapacity);

	this.keyExtractor = keyExtractor;

	this.totalNumberOfKeyGroups = totalNumberOfKeyGroups;
	this.keyGroupRange = keyGroupRange;

	final int keyGroupsInLocalRange = keyGroupRange.getNumberOfKeyGroups();
	final int deduplicationSetSize = 1 + minimumCapacity / keyGroupsInLocalRange;
	this.deduplicationMapsByKeyGroup = new HashMap[keyGroupsInLocalRange];
	for (int i = 0; i < keyGroupsInLocalRange; ++i) {
		deduplicationMapsByKeyGroup[i] = new HashMap<>(deduplicationSetSize);
	}
}
 
Example #9
Source File: RocksDBSnapshotStrategyBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public RocksDBSnapshotStrategyBase(
	@Nonnull String description,
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry) {

	super(description);
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.keySerializer = keySerializer;
	this.kvStateInformation = kvStateInformation;
	this.keyGroupRange = keyGroupRange;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.localRecoveryConfig = localRecoveryConfig;
	this.cancelStreamRegistry = cancelStreamRegistry;
}
 
Example #10
Source File: RocksFullSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public RocksFullSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull StreamCompressionDecorator keyGroupCompressionDecorator) {
	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
}
 
Example #11
Source File: StackTraceSampleService.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Returns a future that completes with a given number of stack trace samples of a task thread.
 *
 * @param task                The task to be sampled from.
 * @param numSamples          The number of samples.
 * @param delayBetweenSamples The time to wait between taking samples.
 * @param maxStackTraceDepth  The maximum depth of the returned stack traces.
 *                            Negative means unlimited.
 * @return A future containing the stack trace samples.
 */
public CompletableFuture<List<StackTraceElement[]>> requestStackTraceSample(
		final StackTraceSampleableTask task,
		@Nonnegative final int numSamples,
		final Time delayBetweenSamples,
		final int maxStackTraceDepth) {

	checkNotNull(task, "task must not be null");
	checkArgument(numSamples > 0, "numSamples must be positive");
	checkNotNull(delayBetweenSamples, "delayBetweenSamples must not be null");

	return requestStackTraceSample(
		task,
		numSamples,
		delayBetweenSamples,
		maxStackTraceDepth,
		new ArrayList<>(numSamples),
		new CompletableFuture<>());
}
 
Example #12
Source File: HeapPriorityQueueSet.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates an empty {@link HeapPriorityQueueSet} with the requested initial capacity.
 *
 * @param elementPriorityComparator comparator for the priority of contained elements.
 * @param keyExtractor function to extract a key from the contained elements.
 * @param minimumCapacity the minimum and initial capacity of this priority queue.
 * @param keyGroupRange the key-group range of the elements in this set.
 * @param totalNumberOfKeyGroups the total number of key-groups of the job.
 */
@SuppressWarnings("unchecked")
public HeapPriorityQueueSet(
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnull KeyExtractorFunction<T> keyExtractor,
	@Nonnegative int minimumCapacity,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalNumberOfKeyGroups) {

	super(elementPriorityComparator, minimumCapacity);

	this.keyExtractor = keyExtractor;

	this.totalNumberOfKeyGroups = totalNumberOfKeyGroups;
	this.keyGroupRange = keyGroupRange;

	final int keyGroupsInLocalRange = keyGroupRange.getNumberOfKeyGroups();
	final int deduplicationSetSize = 1 + minimumCapacity / keyGroupsInLocalRange;
	this.deduplicationMapsByKeyGroup = new HashMap[keyGroupsInLocalRange];
	for (int i = 0; i < keyGroupsInLocalRange; ++i) {
		deduplicationMapsByKeyGroup[i] = new HashMap<>(deduplicationSetSize);
	}
}
 
Example #13
Source File: StackTraceSampleService.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Returns a future that completes with a given number of stack trace samples of a task thread.
 *
 * @param task                The task to be sampled from.
 * @param numSamples          The number of samples.
 * @param delayBetweenSamples The time to wait between taking samples.
 * @param maxStackTraceDepth  The maximum depth of the returned stack traces.
 *                            Negative means unlimited.
 * @return A future containing the stack trace samples.
 */
public CompletableFuture<List<StackTraceElement[]>> requestStackTraceSample(
		final StackTraceSampleableTask task,
		@Nonnegative final int numSamples,
		final Time delayBetweenSamples,
		final int maxStackTraceDepth) {

	checkNotNull(task, "task must not be null");
	checkArgument(numSamples > 0, "numSamples must be positive");
	checkNotNull(delayBetweenSamples, "delayBetweenSamples must not be null");

	return requestStackTraceSample(
		task,
		numSamples,
		delayBetweenSamples,
		maxStackTraceDepth,
		new ArrayList<>(numSamples),
		new CompletableFuture<>());
}
 
Example #14
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up
public RocksFullSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull StreamCompressionDecorator keyGroupCompressionDecorator) {
	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
}
 
Example #15
Source File: CopyOnWriteStateTableSnapshot.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
StateTableKeyGroupPartitioner(
	@Nonnull CopyOnWriteStateTable.StateTableEntry<K, N, S>[] snapshotData,
	@Nonnegative int stateTableSize,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups,
	@Nonnull ElementWriterFunction<CopyOnWriteStateTable.StateTableEntry<K, N, S>> elementWriterFunction) {

	super(
		new CopyOnWriteStateTable.StateTableEntry[stateTableSize],
		stateTableSize,
		// We have made sure that the snapshotData is big enough to hold the flattened entries in
		// CopyOnWriteStateTable#snapshotTableArrays(), we can safely reuse it as the destination array here.
		snapshotData,
		keyGroupRange,
		totalKeyGroups,
		CopyOnWriteStateTable.StateTableEntry::getKey,
		elementWriterFunction);
}
 
Example #16
Source File: RocksDBCachingPriorityQueueSet.java    From flink with Apache License 2.0 6 votes vote down vote up
RocksDBCachingPriorityQueueSet(
	@Nonnegative int keyGroupId,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull RocksDB db,
	@Nonnull ColumnFamilyHandle columnFamilyHandle,
	@Nonnull TypeSerializer<E> byteOrderProducingSerializer,
	@Nonnull DataOutputSerializer outputStream,
	@Nonnull DataInputDeserializer inputStream,
	@Nonnull RocksDBWriteBatchWrapper batchWrapper,
	@Nonnull OrderedByteArraySetCache orderedByteArraySetCache) {
	this.db = db;
	this.columnFamilyHandle = columnFamilyHandle;
	this.byteOrderProducingSerializer = byteOrderProducingSerializer;
	this.batchWrapper = batchWrapper;
	this.outputView = outputStream;
	this.inputView = inputStream;
	this.orderedCache = orderedByteArraySetCache;
	this.allElementsInCache = false;
	this.groupPrefixBytes = createKeyGroupBytes(keyGroupId, keyGroupPrefixBytes);
	this.seekHint = groupPrefixBytes;
	this.internalIndex = HeapPriorityQueueElement.NOT_CONTAINED;
}
 
Example #17
Source File: RocksDBMapState.java    From flink with Apache License 2.0 6 votes vote down vote up
RocksDBMapEntry(
		@Nonnull final RocksDB db,
		@Nonnegative final int userKeyOffset,
		@Nonnull final byte[] rawKeyBytes,
		@Nonnull final byte[] rawValueBytes,
		@Nonnull final TypeSerializer<UK> keySerializer,
		@Nonnull final TypeSerializer<UV> valueSerializer,
		@Nonnull DataInputDeserializer dataInputView) {
	this.db = db;

	this.userKeyOffset = userKeyOffset;
	this.keySerializer = keySerializer;
	this.valueSerializer = valueSerializer;

	this.rawKeyBytes = rawKeyBytes;
	this.rawValueBytes = rawValueBytes;
	this.deleted = false;
	this.dataInputView = dataInputView;
}
 
Example #18
Source File: TaskLocalStateStoreImpl.java    From flink with Apache License 2.0 6 votes vote down vote up
public TaskLocalStateStoreImpl(
	@Nonnull JobID jobID,
	@Nonnull AllocationID allocationID,
	@Nonnull JobVertexID jobVertexID,
	@Nonnegative int subtaskIndex,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull Executor discardExecutor) {

	this(
		jobID,
		allocationID,
		jobVertexID,
		subtaskIndex,
		localRecoveryConfig,
		discardExecutor,
		new TreeMap<>(),
		new Object());
}
 
Example #19
Source File: TaskLocalStateStoreImpl.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public TaskLocalStateStoreImpl(
	@Nonnull JobID jobID,
	@Nonnull AllocationID allocationID,
	@Nonnull JobVertexID jobVertexID,
	@Nonnegative int subtaskIndex,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull Executor discardExecutor) {

	this(
		jobID,
		allocationID,
		jobVertexID,
		subtaskIndex,
		localRecoveryConfig,
		discardExecutor,
		new TreeMap<>(),
		new Object());
}
 
Example #20
Source File: KeyGroupPartitioner.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new {@link KeyGroupPartitioner}.
 *
 * @param partitioningSource the input for the partitioning. All elements must be densely packed in the index
 *                              interval [0, {@link #numberOfElements}[, without null values.
 * @param numberOfElements the number of elements to consider from the input, starting at input index 0.
 * @param partitioningDestination the output of the partitioning. Must have capacity of at least numberOfElements.
 * @param keyGroupRange the key-group range of the data that will be partitioned by this instance.
 * @param totalKeyGroups the total number of key groups in the job.
 * @param keyExtractorFunction this function extracts the partition key from an element.
 */
public KeyGroupPartitioner(
	@Nonnull T[] partitioningSource,
	@Nonnegative int numberOfElements,
	@Nonnull T[] partitioningDestination,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups,
	@Nonnull KeyExtractorFunction<T> keyExtractorFunction,
	@Nonnull ElementWriterFunction<T> elementWriterFunction) {

	Preconditions.checkState(partitioningSource != partitioningDestination);
	Preconditions.checkState(partitioningSource.length >= numberOfElements);
	Preconditions.checkState(partitioningDestination.length >= numberOfElements);

	this.partitioningSource = partitioningSource;
	this.partitioningDestination = partitioningDestination;
	this.numberOfElements = numberOfElements;
	this.keyGroupRange = keyGroupRange;
	this.totalKeyGroups = totalKeyGroups;
	this.keyExtractorFunction = keyExtractorFunction;
	this.elementWriterFunction = elementWriterFunction;
	this.firstKeyGroup = keyGroupRange.getStartKeyGroup();
	this.elementKeyGroups = new int[numberOfElements];
	this.counterHistogram = new int[keyGroupRange.getNumberOfKeyGroups()];
	this.computedResult = null;
}
 
Example #21
Source File: LocalRecoveryDirectoryProviderImpl.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public LocalRecoveryDirectoryProviderImpl(
	@Nonnull File[] allocationBaseDirs,
	@Nonnull JobID jobID,
	@Nonnull JobVertexID jobVertexID,
	@Nonnegative int subtaskIndex) {

	Preconditions.checkArgument(allocationBaseDirs.length > 0);
	this.allocationBaseDirs = allocationBaseDirs;
	this.jobID = jobID;
	this.jobVertexID = jobVertexID;
	this.subtaskIndex = subtaskIndex;

	for (File allocationBaseDir : allocationBaseDirs) {
		Preconditions.checkNotNull(allocationBaseDir);
		allocationBaseDir.mkdirs();
	}
}
 
Example #22
Source File: KeyGroupPartitioner.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new {@link KeyGroupPartitioner}.
 *
 * @param partitioningSource the input for the partitioning. All elements must be densely packed in the index
 *                              interval [0, {@link #numberOfElements}[, without null values.
 * @param numberOfElements the number of elements to consider from the input, starting at input index 0.
 * @param partitioningDestination the output of the partitioning. Must have capacity of at least numberOfElements.
 * @param keyGroupRange the key-group range of the data that will be partitioned by this instance.
 * @param totalKeyGroups the total number of key groups in the job.
 * @param keyExtractorFunction this function extracts the partition key from an element.
 */
public KeyGroupPartitioner(
	@Nonnull T[] partitioningSource,
	@Nonnegative int numberOfElements,
	@Nonnull T[] partitioningDestination,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups,
	@Nonnull KeyExtractorFunction<T> keyExtractorFunction,
	@Nonnull ElementWriterFunction<T> elementWriterFunction) {

	Preconditions.checkState(partitioningSource != partitioningDestination);
	Preconditions.checkState(partitioningSource.length >= numberOfElements);
	Preconditions.checkState(partitioningDestination.length >= numberOfElements);

	this.partitioningSource = partitioningSource;
	this.partitioningDestination = partitioningDestination;
	this.numberOfElements = numberOfElements;
	this.keyGroupRange = keyGroupRange;
	this.totalKeyGroups = totalKeyGroups;
	this.keyExtractorFunction = keyExtractorFunction;
	this.elementWriterFunction = elementWriterFunction;
	this.firstKeyGroup = keyGroupRange.getStartKeyGroup();
	this.elementKeyGroups = new int[numberOfElements];
	this.counterHistogram = new int[keyGroupRange.getNumberOfKeyGroups()];
	this.computedResult = null;
}
 
Example #23
Source File: HeapPriorityQueueSetFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
public HeapPriorityQueueSetFactory(
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int totalKeyGroups,
	@Nonnegative int minimumCapacity) {

	this.keyGroupRange = keyGroupRange;
	this.totalKeyGroups = totalKeyGroups;
	this.minimumCapacity = minimumCapacity;
}
 
Example #24
Source File: RocksDBSerializedCompositeKeyBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Sets the key and key-group as prefix. This will serialize them into the buffer and the will be used to create
 * composite keys with provided namespaces.
 *
 * @param key        the key.
 * @param keyGroupId the key-group id for the key.
 */
public void setKeyAndKeyGroup(@Nonnull K key, @Nonnegative int keyGroupId) {
	try {
		serializeKeyGroupAndKey(key, keyGroupId);
	} catch (IOException shouldNeverHappen) {
		throw new FlinkRuntimeException(shouldNeverHappen);
	}
}
 
Example #25
Source File: KeyGroupPartitionerTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
ValidatingElementWriterDummy(
	@Nonnull KeyExtractorFunction<T> keyExtractorFunction,
	@Nonnegative int numberOfKeyGroups,
	@Nonnull Set<T> allElementsSet) {
	this.keyExtractorFunction = keyExtractorFunction;
	this.numberOfKeyGroups = numberOfKeyGroups;
	this.allElementsSet = allElementsSet;
}
 
Example #26
Source File: CheckpointStreamWithResultProvider.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
static CheckpointStreamWithResultProvider createDuplicatingStream(
	@Nonnegative long checkpointId,
	@Nonnull CheckpointedStateScope checkpointedStateScope,
	@Nonnull CheckpointStreamFactory primaryStreamFactory,
	@Nonnull LocalRecoveryDirectoryProvider secondaryStreamDirProvider) throws IOException {

	CheckpointStreamFactory.CheckpointStateOutputStream primaryOut =
		primaryStreamFactory.createCheckpointStateOutputStream(checkpointedStateScope);

	try {
		File outFile = new File(
			secondaryStreamDirProvider.subtaskSpecificCheckpointDirectory(checkpointId),
			String.valueOf(UUID.randomUUID()));
		Path outPath = new Path(outFile.toURI());

		CheckpointStreamFactory.CheckpointStateOutputStream secondaryOut =
			new FileBasedStateOutputStream(outPath.getFileSystem(), outPath);

		return new CheckpointStreamWithResultProvider.PrimaryAndSecondaryStream(primaryOut, secondaryOut);
	} catch (IOException secondaryEx) {
		LOG.warn("Exception when opening secondary/local checkpoint output stream. " +
			"Continue only with the primary stream.", secondaryEx);
	}

	return new CheckpointStreamWithResultProvider.PrimaryStreamOnly(primaryOut);
}
 
Example #27
Source File: KeyGroupPartitioner.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
PartitioningResult(
	@Nonnull ElementWriterFunction<T> elementWriterFunction,
	@Nonnegative int firstKeyGroup,
	@Nonnull int[] keyGroupEndOffsets,
	@Nonnull T[] partitionedElements) {
	this.elementWriterFunction = elementWriterFunction;
	this.firstKeyGroup = firstKeyGroup;
	this.keyGroupOffsets = keyGroupEndOffsets;
	this.partitionedElements = partitionedElements;
}
 
Example #28
Source File: IncrementalLocalKeyedStateHandle.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public IncrementalLocalKeyedStateHandle(
	@Nonnull UUID backendIdentifier,
	@Nonnegative long checkpointId,
	@Nonnull DirectoryStateHandle directoryStateHandle,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnull StreamStateHandle metaDataState,
	@Nonnull Set<StateHandleID> sharedStateHandleIDs) {

	super(directoryStateHandle, keyGroupRange);
	this.backendIdentifier = backendIdentifier;
	this.checkpointId = checkpointId;
	this.metaDataState = metaDataState;
	this.sharedStateHandleIDs = sharedStateHandleIDs;
}
 
Example #29
Source File: IncrementalLocalKeyedStateHandle.java    From flink with Apache License 2.0 5 votes vote down vote up
public IncrementalLocalKeyedStateHandle(
	@Nonnull UUID backendIdentifier,
	@Nonnegative long checkpointId,
	@Nonnull DirectoryStateHandle directoryStateHandle,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnull StreamStateHandle metaDataState,
	@Nonnull Set<StateHandleID> sharedStateHandleIDs) {

	super(directoryStateHandle, keyGroupRange);
	this.backendIdentifier = backendIdentifier;
	this.checkpointId = checkpointId;
	this.metaDataState = metaDataState;
	this.sharedStateHandleIDs = sharedStateHandleIDs;
}
 
Example #30
Source File: HeapPriorityQueue.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates an empty {@link HeapPriorityQueue} with the requested initial capacity.
 *
 * @param elementPriorityComparator comparator for the priority of contained elements.
 * @param minimumCapacity the minimum and initial capacity of this priority queue.
 */
@SuppressWarnings("unchecked")
public HeapPriorityQueue(
	@Nonnull PriorityComparator<T> elementPriorityComparator,
	@Nonnegative int minimumCapacity) {
	super(minimumCapacity);
	this.elementPriorityComparator = elementPriorityComparator;
}