Java Code Examples for org.apache.flink.streaming.connectors.kinesis.testutils.TestableKinesisDataFetcher#discoverNewShardsToSubscribe()

The following examples show how to use org.apache.flink.streaming.connectors.kinesis.testutils.TestableKinesisDataFetcher#discoverNewShardsToSubscribe() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KinesisDataFetcherTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testShardToSubtaskMappingWithCustomHashFunction() throws Exception {

	int totalCountOfSubtasks = 10;
	int shardCount = 3;

	for (int i = 0; i < 2; i++) {

		final int hash = i;
		final KinesisShardAssigner allShardsSingleSubtaskFn = (shard, subtasks) -> hash;
		Map<String, Integer> streamToShardCount = new HashMap<>();
		List<String> fakeStreams = new LinkedList<>();
		fakeStreams.add("fakeStream");
		streamToShardCount.put("fakeStream", shardCount);

		for (int j = 0; j < totalCountOfSubtasks; j++) {

			int subtaskIndex = j;
			// subscribe with default hashing
			final TestableKinesisDataFetcher fetcher =
				new TestableKinesisDataFetcher(
					fakeStreams,
					new TestSourceContext<>(),
					new Properties(),
					new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
					totalCountOfSubtasks,
					subtaskIndex,
					new AtomicReference<>(),
					new LinkedList<>(),
					KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams),
					FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
			Whitebox.setInternalState(fetcher, "shardAssigner", allShardsSingleSubtaskFn); // override hashing
			List<StreamShardHandle> shards = fetcher.discoverNewShardsToSubscribe();
			fetcher.shutdownFetcher();

			String msg = String.format("for hash=%d, subtask=%d", hash, subtaskIndex);
			if (j == i) {
				assertEquals(msg, shardCount, shards.size());
			} else {
				assertEquals(msg, 0, shards.size());
			}
		}

	}

}
Example 2
Source File: KinesisDataFetcherTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testShardToSubtaskMappingWithCustomHashFunction() throws Exception {

	int totalCountOfSubtasks = 10;
	int shardCount = 3;

	for (int i = 0; i < 2; i++) {

		final int hash = i;
		final KinesisShardAssigner allShardsSingleSubtaskFn = (shard, subtasks) -> hash;
		Map<String, Integer> streamToShardCount = new HashMap<>();
		List<String> fakeStreams = new LinkedList<>();
		fakeStreams.add("fakeStream");
		streamToShardCount.put("fakeStream", shardCount);

		for (int j = 0; j < totalCountOfSubtasks; j++) {

			int subtaskIndex = j;
			// subscribe with default hashing
			final TestableKinesisDataFetcher fetcher =
				new TestableKinesisDataFetcher(
					fakeStreams,
					new TestSourceContext<>(),
					new Properties(),
					new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
					totalCountOfSubtasks,
					subtaskIndex,
					new AtomicReference<>(),
					new LinkedList<>(),
					KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams),
					FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
			Whitebox.setInternalState(fetcher, "shardAssigner", allShardsSingleSubtaskFn); // override hashing
			List<StreamShardHandle> shards = fetcher.discoverNewShardsToSubscribe();
			fetcher.shutdownFetcher();

			String msg = String.format("for hash=%d, subtask=%d", hash, subtaskIndex);
			if (j == i) {
				assertEquals(msg, shardCount, shards.size());
			} else {
				assertEquals(msg, 0, shards.size());
			}
		}

	}

}
Example 3
Source File: KinesisDataFetcherTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testShardToSubtaskMappingWithCustomHashFunction() throws Exception {

	int totalCountOfSubtasks = 10;
	int shardCount = 3;

	for (int i = 0; i < 2; i++) {

		final int hash = i;
		final KinesisShardAssigner allShardsSingleSubtaskFn = (shard, subtasks) -> hash;
		Map<String, Integer> streamToShardCount = new HashMap<>();
		List<String> fakeStreams = new LinkedList<>();
		fakeStreams.add("fakeStream");
		streamToShardCount.put("fakeStream", shardCount);

		for (int j = 0; j < totalCountOfSubtasks; j++) {

			int subtaskIndex = j;
			// subscribe with default hashing
			final TestableKinesisDataFetcher fetcher =
				new TestableKinesisDataFetcher(
					fakeStreams,
					new TestSourceContext<>(),
					new Properties(),
					new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
					totalCountOfSubtasks,
					subtaskIndex,
					new AtomicReference<>(),
					new LinkedList<>(),
					KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams),
					FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
			Whitebox.setInternalState(fetcher, "shardAssigner", allShardsSingleSubtaskFn); // override hashing
			List<StreamShardHandle> shards = fetcher.discoverNewShardsToSubscribe();
			fetcher.shutdownFetcher();

			String msg = String.format("for hash=%d, subtask=%d", hash, subtaskIndex);
			if (j == i) {
				assertEquals(msg, shardCount, shards.size());
			} else {
				assertEquals(msg, 0, shards.size());
			}
		}

	}

}