org.apache.flink.streaming.connectors.kinesis.KinesisShardAssigner Java Examples

The following examples show how to use org.apache.flink.streaming.connectors.kinesis.KinesisShardAssigner. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DynamoDBStreamsDataFetcher.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Constructor.
 *
 * @param streams list of streams to fetch data
 * @param sourceContext source context
 * @param runtimeContext runtime context
 * @param configProps config properties
 * @param deserializationSchema deserialization schema
 * @param shardAssigner shard assigner
 */
public DynamoDBStreamsDataFetcher(List<String> streams,
	SourceFunction.SourceContext<T> sourceContext,
	RuntimeContext runtimeContext,
	Properties configProps,
	KinesisDeserializationSchema<T> deserializationSchema,
	KinesisShardAssigner shardAssigner) {

	super(streams,
		sourceContext,
		sourceContext.getCheckpointLock(),
		runtimeContext,
		configProps,
		deserializationSchema,
		shardAssigner,
		null,
		null,
		new AtomicReference<>(),
		new ArrayList<>(),
		createInitialSubscribedStreamsToLastDiscoveredShardsState(streams),
		// use DynamoDBStreamsProxy
		DynamoDBStreamsProxy::create);
}
Example #2
Source File: KinesisDataFetcher.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Creates a Kinesis Data Fetcher.
 *
 * @param streams the streams to subscribe to
 * @param sourceContext context of the source function
 * @param runtimeContext this subtask's runtime context
 * @param configProps the consumer configuration properties
 * @param deserializationSchema deserialization schema
 */
public KinesisDataFetcher(List<String> streams,
						SourceFunction.SourceContext<T> sourceContext,
						RuntimeContext runtimeContext,
						Properties configProps,
						KinesisDeserializationSchema<T> deserializationSchema,
						KinesisShardAssigner shardAssigner,
						AssignerWithPeriodicWatermarks<T> periodicWatermarkAssigner,
						WatermarkTracker watermarkTracker) {
	this(streams,
		sourceContext,
		sourceContext.getCheckpointLock(),
		runtimeContext,
		configProps,
		deserializationSchema,
		shardAssigner,
		periodicWatermarkAssigner,
		watermarkTracker,
		new AtomicReference<>(),
		new ArrayList<>(),
		createInitialSubscribedStreamsToLastDiscoveredShardsState(streams),
		KinesisProxy::create);
}
Example #3
Source File: DynamoDBStreamsDataFetcher.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Constructor.
 *
 * @param streams list of streams to fetch data
 * @param sourceContext source context
 * @param runtimeContext runtime context
 * @param configProps config properties
 * @param deserializationSchema deserialization schema
 * @param shardAssigner shard assigner
 */
public DynamoDBStreamsDataFetcher(List<String> streams,
	SourceFunction.SourceContext<T> sourceContext,
	RuntimeContext runtimeContext,
	Properties configProps,
	KinesisDeserializationSchema<T> deserializationSchema,
	KinesisShardAssigner shardAssigner) {

	super(streams,
		sourceContext,
		sourceContext.getCheckpointLock(),
		runtimeContext,
		configProps,
		deserializationSchema,
		shardAssigner,
		null,
		null,
		new AtomicReference<>(),
		new ArrayList<>(),
		createInitialSubscribedStreamsToLastDiscoveredShardsState(streams),
		// use DynamoDBStreamsProxy
		DynamoDBStreamsProxy::create);
}
Example #4
Source File: KinesisDataFetcher.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Creates a Kinesis Data Fetcher.
 *
 * @param streams the streams to subscribe to
 * @param sourceContext context of the source function
 * @param runtimeContext this subtask's runtime context
 * @param configProps the consumer configuration properties
 * @param deserializationSchema deserialization schema
 */
public KinesisDataFetcher(List<String> streams,
						SourceFunction.SourceContext<T> sourceContext,
						RuntimeContext runtimeContext,
						Properties configProps,
						KinesisDeserializationSchema<T> deserializationSchema,
						KinesisShardAssigner shardAssigner,
						AssignerWithPeriodicWatermarks<T> periodicWatermarkAssigner,
						WatermarkTracker watermarkTracker) {
	this(streams,
		sourceContext,
		sourceContext.getCheckpointLock(),
		runtimeContext,
		configProps,
		deserializationSchema,
		shardAssigner,
		periodicWatermarkAssigner,
		watermarkTracker,
		new AtomicReference<>(),
		new ArrayList<>(),
		createInitialSubscribedStreamsToLastDiscoveredShardsState(streams),
		KinesisProxy::create);
}
Example #5
Source File: DynamoDBStreamsDataFetcher.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Constructor.
 *
 * @param streams list of streams to fetch data
 * @param sourceContext source context
 * @param runtimeContext runtime context
 * @param configProps config properties
 * @param deserializationSchema deserialization schema
 * @param shardAssigner shard assigner
 */
public DynamoDBStreamsDataFetcher(List<String> streams,
	SourceFunction.SourceContext<T> sourceContext,
	RuntimeContext runtimeContext,
	Properties configProps,
	KinesisDeserializationSchema<T> deserializationSchema,
	KinesisShardAssigner shardAssigner) {

	super(streams,
		sourceContext,
		sourceContext.getCheckpointLock(),
		runtimeContext,
		configProps,
		deserializationSchema,
		shardAssigner,
		null,
		null,
		new AtomicReference<>(),
		new ArrayList<>(),
		createInitialSubscribedStreamsToLastDiscoveredShardsState(streams),
		// use DynamoDBStreamsProxy
		DynamoDBStreamsProxy::create);
}
Example #6
Source File: KinesisDataFetcher.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Creates a Kinesis Data Fetcher.
 *
 * @param streams the streams to subscribe to
 * @param sourceContext context of the source function
 * @param runtimeContext this subtask's runtime context
 * @param configProps the consumer configuration properties
 * @param deserializationSchema deserialization schema
 */
public KinesisDataFetcher(List<String> streams,
						SourceFunction.SourceContext<T> sourceContext,
						RuntimeContext runtimeContext,
						Properties configProps,
						KinesisDeserializationSchema<T> deserializationSchema,
						KinesisShardAssigner shardAssigner,
						AssignerWithPeriodicWatermarks<T> periodicWatermarkAssigner,
						WatermarkTracker watermarkTracker) {
	this(streams,
		sourceContext,
		sourceContext.getCheckpointLock(),
		runtimeContext,
		configProps,
		deserializationSchema,
		shardAssigner,
		periodicWatermarkAssigner,
		watermarkTracker,
		new AtomicReference<>(),
		new ArrayList<>(),
		createInitialSubscribedStreamsToLastDiscoveredShardsState(streams),
		KinesisProxy::create);
}
Example #7
Source File: KinesisDataFetcher.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
protected KinesisDataFetcher(List<String> streams,
							SourceFunction.SourceContext<T> sourceContext,
							Object checkpointLock,
							RuntimeContext runtimeContext,
							Properties configProps,
							KinesisDeserializationSchema<T> deserializationSchema,
							KinesisShardAssigner shardAssigner,
							AssignerWithPeriodicWatermarks<T> periodicWatermarkAssigner,
							WatermarkTracker watermarkTracker,
							AtomicReference<Throwable> error,
							List<KinesisStreamShardState> subscribedShardsState,
							HashMap<String, String> subscribedStreamsToLastDiscoveredShardIds,
							FlinkKinesisProxyFactory kinesisProxyFactory) {
	this.streams = checkNotNull(streams);
	this.configProps = checkNotNull(configProps);
	this.sourceContext = checkNotNull(sourceContext);
	this.checkpointLock = checkNotNull(checkpointLock);
	this.runtimeContext = checkNotNull(runtimeContext);
	this.totalNumberOfConsumerSubtasks = runtimeContext.getNumberOfParallelSubtasks();
	this.indexOfThisConsumerSubtask = runtimeContext.getIndexOfThisSubtask();
	this.deserializationSchema = checkNotNull(deserializationSchema);
	this.shardAssigner = checkNotNull(shardAssigner);
	this.periodicWatermarkAssigner = periodicWatermarkAssigner;
	this.watermarkTracker = watermarkTracker;
	this.kinesisProxyFactory = checkNotNull(kinesisProxyFactory);
	this.kinesis = kinesisProxyFactory.create(configProps);

	this.consumerMetricGroup = runtimeContext.getMetricGroup()
		.addGroup(KinesisConsumerMetricConstants.KINESIS_CONSUMER_METRICS_GROUP);

	this.error = checkNotNull(error);
	this.subscribedShardsState = checkNotNull(subscribedShardsState);
	this.subscribedStreamsToLastDiscoveredShardIds = checkNotNull(subscribedStreamsToLastDiscoveredShardIds);

	this.shardConsumersExecutor =
		createShardConsumersThreadPool(runtimeContext.getTaskNameWithSubtasks());
	this.recordEmitter = createRecordEmitter(configProps);
}
Example #8
Source File: KinesisDataFetcher.java    From flink with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
protected KinesisDataFetcher(List<String> streams,
							SourceFunction.SourceContext<T> sourceContext,
							Object checkpointLock,
							RuntimeContext runtimeContext,
							Properties configProps,
							KinesisDeserializationSchema<T> deserializationSchema,
							KinesisShardAssigner shardAssigner,
							AssignerWithPeriodicWatermarks<T> periodicWatermarkAssigner,
							WatermarkTracker watermarkTracker,
							AtomicReference<Throwable> error,
							List<KinesisStreamShardState> subscribedShardsState,
							HashMap<String, String> subscribedStreamsToLastDiscoveredShardIds,
							FlinkKinesisProxyFactory kinesisProxyFactory) {
	this.streams = checkNotNull(streams);
	this.configProps = checkNotNull(configProps);
	this.sourceContext = checkNotNull(sourceContext);
	this.checkpointLock = checkNotNull(checkpointLock);
	this.runtimeContext = checkNotNull(runtimeContext);
	this.totalNumberOfConsumerSubtasks = runtimeContext.getNumberOfParallelSubtasks();
	this.indexOfThisConsumerSubtask = runtimeContext.getIndexOfThisSubtask();
	this.deserializationSchema = checkNotNull(deserializationSchema);
	this.shardAssigner = checkNotNull(shardAssigner);
	this.periodicWatermarkAssigner = periodicWatermarkAssigner;
	this.watermarkTracker = watermarkTracker;
	this.kinesisProxyFactory = checkNotNull(kinesisProxyFactory);
	this.kinesis = kinesisProxyFactory.create(configProps);

	this.consumerMetricGroup = runtimeContext.getMetricGroup()
		.addGroup(KinesisConsumerMetricConstants.KINESIS_CONSUMER_METRICS_GROUP);

	this.error = checkNotNull(error);
	this.subscribedShardsState = checkNotNull(subscribedShardsState);
	this.subscribedStreamsToLastDiscoveredShardIds = checkNotNull(subscribedStreamsToLastDiscoveredShardIds);

	this.shardConsumersExecutor =
		createShardConsumersThreadPool(runtimeContext.getTaskNameWithSubtasks());
	this.recordEmitter = createRecordEmitter(configProps);
}
Example #9
Source File: KinesisDataFetcher.java    From flink with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
protected KinesisDataFetcher(List<String> streams,
							SourceFunction.SourceContext<T> sourceContext,
							Object checkpointLock,
							RuntimeContext runtimeContext,
							Properties configProps,
							KinesisDeserializationSchema<T> deserializationSchema,
							KinesisShardAssigner shardAssigner,
							AssignerWithPeriodicWatermarks<T> periodicWatermarkAssigner,
							WatermarkTracker watermarkTracker,
							AtomicReference<Throwable> error,
							List<KinesisStreamShardState> subscribedShardsState,
							HashMap<String, String> subscribedStreamsToLastDiscoveredShardIds,
							FlinkKinesisProxyFactory kinesisProxyFactory) {
	this.streams = checkNotNull(streams);
	this.configProps = checkNotNull(configProps);
	this.sourceContext = checkNotNull(sourceContext);
	this.checkpointLock = checkNotNull(checkpointLock);
	this.runtimeContext = checkNotNull(runtimeContext);
	this.totalNumberOfConsumerSubtasks = runtimeContext.getNumberOfParallelSubtasks();
	this.indexOfThisConsumerSubtask = runtimeContext.getIndexOfThisSubtask();
	this.deserializationSchema = checkNotNull(deserializationSchema);
	this.shardAssigner = checkNotNull(shardAssigner);
	this.periodicWatermarkAssigner = periodicWatermarkAssigner;
	this.watermarkTracker = watermarkTracker;
	this.kinesisProxyFactory = checkNotNull(kinesisProxyFactory);
	this.kinesis = kinesisProxyFactory.create(configProps);

	this.consumerMetricGroup = runtimeContext.getMetricGroup()
		.addGroup(KinesisConsumerMetricConstants.KINESIS_CONSUMER_METRICS_GROUP);

	this.error = checkNotNull(error);
	this.subscribedShardsState = checkNotNull(subscribedShardsState);
	this.subscribedStreamsToLastDiscoveredShardIds = checkNotNull(subscribedStreamsToLastDiscoveredShardIds);

	this.shardConsumersExecutor =
		createShardConsumersThreadPool(runtimeContext.getTaskNameWithSubtasks());
	this.recordEmitter = createRecordEmitter(configProps);
}
Example #10
Source File: KinesisDataFetcherTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testShardToSubtaskMappingWithCustomHashFunction() throws Exception {

	int totalCountOfSubtasks = 10;
	int shardCount = 3;

	for (int i = 0; i < 2; i++) {

		final int hash = i;
		final KinesisShardAssigner allShardsSingleSubtaskFn = (shard, subtasks) -> hash;
		Map<String, Integer> streamToShardCount = new HashMap<>();
		List<String> fakeStreams = new LinkedList<>();
		fakeStreams.add("fakeStream");
		streamToShardCount.put("fakeStream", shardCount);

		for (int j = 0; j < totalCountOfSubtasks; j++) {

			int subtaskIndex = j;
			// subscribe with default hashing
			final TestableKinesisDataFetcher fetcher =
				new TestableKinesisDataFetcher(
					fakeStreams,
					new TestSourceContext<>(),
					new Properties(),
					new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
					totalCountOfSubtasks,
					subtaskIndex,
					new AtomicReference<>(),
					new LinkedList<>(),
					KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams),
					FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
			Whitebox.setInternalState(fetcher, "shardAssigner", allShardsSingleSubtaskFn); // override hashing
			List<StreamShardHandle> shards = fetcher.discoverNewShardsToSubscribe();
			fetcher.shutdownFetcher();

			String msg = String.format("for hash=%d, subtask=%d", hash, subtaskIndex);
			if (j == i) {
				assertEquals(msg, shardCount, shards.size());
			} else {
				assertEquals(msg, 0, shards.size());
			}
		}

	}

}
Example #11
Source File: KinesisDataFetcherTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testShardToSubtaskMappingWithCustomHashFunction() throws Exception {

	int totalCountOfSubtasks = 10;
	int shardCount = 3;

	for (int i = 0; i < 2; i++) {

		final int hash = i;
		final KinesisShardAssigner allShardsSingleSubtaskFn = (shard, subtasks) -> hash;
		Map<String, Integer> streamToShardCount = new HashMap<>();
		List<String> fakeStreams = new LinkedList<>();
		fakeStreams.add("fakeStream");
		streamToShardCount.put("fakeStream", shardCount);

		for (int j = 0; j < totalCountOfSubtasks; j++) {

			int subtaskIndex = j;
			// subscribe with default hashing
			final TestableKinesisDataFetcher fetcher =
				new TestableKinesisDataFetcher(
					fakeStreams,
					new TestSourceContext<>(),
					new Properties(),
					new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
					totalCountOfSubtasks,
					subtaskIndex,
					new AtomicReference<>(),
					new LinkedList<>(),
					KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams),
					FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
			Whitebox.setInternalState(fetcher, "shardAssigner", allShardsSingleSubtaskFn); // override hashing
			List<StreamShardHandle> shards = fetcher.discoverNewShardsToSubscribe();
			fetcher.shutdownFetcher();

			String msg = String.format("for hash=%d, subtask=%d", hash, subtaskIndex);
			if (j == i) {
				assertEquals(msg, shardCount, shards.size());
			} else {
				assertEquals(msg, 0, shards.size());
			}
		}

	}

}
Example #12
Source File: KinesisDataFetcherTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testShardToSubtaskMappingWithCustomHashFunction() throws Exception {

	int totalCountOfSubtasks = 10;
	int shardCount = 3;

	for (int i = 0; i < 2; i++) {

		final int hash = i;
		final KinesisShardAssigner allShardsSingleSubtaskFn = (shard, subtasks) -> hash;
		Map<String, Integer> streamToShardCount = new HashMap<>();
		List<String> fakeStreams = new LinkedList<>();
		fakeStreams.add("fakeStream");
		streamToShardCount.put("fakeStream", shardCount);

		for (int j = 0; j < totalCountOfSubtasks; j++) {

			int subtaskIndex = j;
			// subscribe with default hashing
			final TestableKinesisDataFetcher fetcher =
				new TestableKinesisDataFetcher(
					fakeStreams,
					new TestSourceContext<>(),
					new Properties(),
					new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
					totalCountOfSubtasks,
					subtaskIndex,
					new AtomicReference<>(),
					new LinkedList<>(),
					KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams),
					FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
			Whitebox.setInternalState(fetcher, "shardAssigner", allShardsSingleSubtaskFn); // override hashing
			List<StreamShardHandle> shards = fetcher.discoverNewShardsToSubscribe();
			fetcher.shutdownFetcher();

			String msg = String.format("for hash=%d, subtask=%d", hash, subtaskIndex);
			if (j == i) {
				assertEquals(msg, shardCount, shards.size());
			} else {
				assertEquals(msg, 0, shards.size());
			}
		}

	}

}