org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxyInterface Java Examples
The following examples show how to use
org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxyInterface.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: KinesisPubsubClient.java From flink with Apache License 2.0 | 6 votes |
|
public List<String> readAllMessages(String streamName) throws Exception {
KinesisProxyInterface kinesisProxy = KinesisProxy.create(properties);
Map<String, String> streamNamesWithLastSeenShardIds = new HashMap<>();
streamNamesWithLastSeenShardIds.put(streamName, null);
GetShardListResult shardListResult = kinesisProxy.getShardList(streamNamesWithLastSeenShardIds);
int maxRecordsToFetch = 10;
List<String> messages = new ArrayList<>();
// retrieve records from all shards
for (StreamShardHandle ssh : shardListResult.getRetrievedShardListOfStream(streamName)) {
String shardIterator = kinesisProxy.getShardIterator(ssh, "TRIM_HORIZON", null);
GetRecordsResult getRecordsResult = kinesisProxy.getRecords(shardIterator, maxRecordsToFetch);
List<Record> aggregatedRecords = getRecordsResult.getRecords();
for (Record record : aggregatedRecords) {
messages.add(new String(record.getData().array()));
}
}
return messages;
}
Example #2
| Source File: FakeKinesisBehavioursFactory.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
public static KinesisProxyInterface noShardsFoundForRequestedStreamsBehaviour() {
return new KinesisProxyInterface() {
@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
return new GetShardListResult(); // not setting any retrieved shards for result
}
@Override
public String getShardIterator(StreamShardHandle shard, String shardIteratorType, Object startingMarker) {
return null;
}
@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
return null;
}
};
}
Example #3
| Source File: FakeKinesisBehavioursFactory.java From flink with Apache License 2.0 | 6 votes |
|
public static KinesisProxyInterface noShardsFoundForRequestedStreamsBehaviour() {
return new KinesisProxyInterface() {
@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
return new GetShardListResult(); // not setting any retrieved shards for result
}
@Override
public String getShardIterator(StreamShardHandle shard, String shardIteratorType, Object startingMarker) {
return null;
}
@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
return null;
}
};
}
Example #4
| Source File: KinesisPubsubClient.java From flink with Apache License 2.0 | 6 votes |
|
@Override
public List<String> readAllMessages(String streamName) throws Exception {
KinesisProxyInterface kinesisProxy = KinesisProxy.create(properties);
Map<String, String> streamNamesWithLastSeenShardIds = new HashMap<>();
streamNamesWithLastSeenShardIds.put(streamName, null);
GetShardListResult shardListResult = kinesisProxy.getShardList(streamNamesWithLastSeenShardIds);
int maxRecordsToFetch = 10;
List<String> messages = new ArrayList<>();
// retrieve records from all shards
for (StreamShardHandle ssh : shardListResult.getRetrievedShardListOfStream(streamName)) {
String shardIterator = kinesisProxy.getShardIterator(ssh, "TRIM_HORIZON", null);
GetRecordsResult getRecordsResult = kinesisProxy.getRecords(shardIterator, maxRecordsToFetch);
List<Record> aggregatedRecords = getRecordsResult.getRecords();
for (Record record : aggregatedRecords) {
messages.add(new String(record.getData().array()));
}
}
return messages;
}
Example #5
| Source File: KinesisPubsubClient.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Override
public List<String> readAllMessages(String streamName) throws Exception {
KinesisProxyInterface kinesisProxy = KinesisProxy.create(properties);
Map<String, String> streamNamesWithLastSeenShardIds = new HashMap<>();
streamNamesWithLastSeenShardIds.put(streamName, null);
GetShardListResult shardListResult = kinesisProxy.getShardList(streamNamesWithLastSeenShardIds);
int maxRecordsToFetch = 10;
List<String> messages = new ArrayList<>();
// retrieve records from all shards
for (StreamShardHandle ssh : shardListResult.getRetrievedShardListOfStream(streamName)) {
String shardIterator = kinesisProxy.getShardIterator(ssh, "TRIM_HORIZON", null);
GetRecordsResult getRecordsResult = kinesisProxy.getRecords(shardIterator, maxRecordsToFetch);
List<Record> aggregatedRecords = getRecordsResult.getRecords();
for (Record record : aggregatedRecords) {
messages.add(new String(record.getData().array()));
}
}
return messages;
}
Example #6
| Source File: ShardConsumerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testMetricsReporting() {
StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
subscribedShardsStateUnderTest.add(
new KinesisStreamShardState(
KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
fakeToBeConsumedShard,
new SequenceNumber("fakeStartingState")));
TestSourceContext<String> sourceContext = new TestSourceContext<>();
TestableKinesisDataFetcher<String> fetcher =
new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
sourceContext,
new Properties(),
new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
ShardMetricsReporter shardMetricsReporter = new ShardMetricsReporter();
long millisBehindLatest = 500L;
new ShardConsumer<>(
fetcher,
0,
subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, millisBehindLatest),
shardMetricsReporter).run();
// the millisBehindLatest metric should have been reported
assertEquals(millisBehindLatest, shardMetricsReporter.getMillisBehindLatest());
}
Example #7
| Source File: TestableKinesisDataFetcherForShardConsumerException.java From flink with Apache License 2.0 | 5 votes |
|
public TestableKinesisDataFetcherForShardConsumerException(final List<String> fakeStreams,
final SourceFunction.SourceContext<T> sourceContext,
final Properties fakeConfiguration,
final KinesisDeserializationSchema<T> deserializationSchema,
final int fakeTotalCountOfSubtasks,
final int fakeIndexOfThisSubtask,
final AtomicReference<Throwable> thrownErrorUnderTest,
final LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
final HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
final KinesisProxyInterface fakeKinesis) {
super(fakeStreams, sourceContext, fakeConfiguration, deserializationSchema, fakeTotalCountOfSubtasks,
fakeIndexOfThisSubtask, thrownErrorUnderTest, subscribedShardsStateUnderTest,
subscribedStreamsToLastDiscoveredShardIdsStateUnderTest, fakeKinesis);
}
Example #8
| Source File: TestableKinesisDataFetcher.java From flink with Apache License 2.0 | 5 votes |
|
public TestableKinesisDataFetcher(
List<String> fakeStreams,
SourceFunction.SourceContext<T> sourceContext,
Properties fakeConfiguration,
KinesisDeserializationSchema<T> deserializationSchema,
int fakeTotalCountOfSubtasks,
int fakeIndexOfThisSubtask,
AtomicReference<Throwable> thrownErrorUnderTest,
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
KinesisProxyInterface fakeKinesis) {
super(
fakeStreams,
sourceContext,
sourceContext.getCheckpointLock(),
getMockedRuntimeContext(fakeTotalCountOfSubtasks, fakeIndexOfThisSubtask),
fakeConfiguration,
deserializationSchema,
DEFAULT_SHARD_ASSIGNER,
null,
null,
thrownErrorUnderTest,
subscribedShardsStateUnderTest,
subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
(properties) -> fakeKinesis);
this.runWaiter = new OneShotLatch();
this.initialDiscoveryWaiter = new OneShotLatch();
this.shutdownWaiter = new OneShotLatch();
this.running = true;
}
Example #9
| Source File: FakeKinesisBehavioursFactory.java From flink with Apache License 2.0 | 5 votes |
|
public static KinesisProxyInterface initialNumOfRecordsAfterNumOfGetRecordsCallsWithAdaptiveReads(
final int numOfRecords,
final int numOfGetRecordsCalls,
final long millisBehindLatest) {
return new SingleShardEmittingAdaptiveNumOfRecordsKinesis(numOfRecords, numOfGetRecordsCalls,
millisBehindLatest);
}
Example #10
| Source File: FakeKinesisBehavioursFactory.java From flink with Apache License 2.0 | 5 votes |
|
public static KinesisProxyInterface totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator(
final int numOfRecords,
final int numOfGetRecordsCall,
final int orderOfCallToExpire,
final long millisBehindLatest) {
return new SingleShardEmittingFixNumOfRecordsWithExpiredIteratorKinesis(
numOfRecords, numOfGetRecordsCall, orderOfCallToExpire, millisBehindLatest);
}
Example #11
| Source File: TestableKinesisDataFetcherForShardConsumerException.java From flink with Apache License 2.0 | 5 votes |
|
public TestableKinesisDataFetcherForShardConsumerException(final List<String> fakeStreams,
final SourceFunction.SourceContext<T> sourceContext,
final Properties fakeConfiguration,
final KinesisDeserializationSchema<T> deserializationSchema,
final int fakeTotalCountOfSubtasks,
final int fakeIndexOfThisSubtask,
final AtomicReference<Throwable> thrownErrorUnderTest,
final LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
final HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
final KinesisProxyInterface fakeKinesis) {
super(fakeStreams, sourceContext, fakeConfiguration, deserializationSchema, fakeTotalCountOfSubtasks,
fakeIndexOfThisSubtask, thrownErrorUnderTest, subscribedShardsStateUnderTest,
subscribedStreamsToLastDiscoveredShardIdsStateUnderTest, fakeKinesis);
}
Example #12
| Source File: TestableKinesisDataFetcher.java From flink with Apache License 2.0 | 5 votes |
|
public TestableKinesisDataFetcher(
List<String> fakeStreams,
SourceFunction.SourceContext<T> sourceContext,
Properties fakeConfiguration,
KinesisDeserializationSchema<T> deserializationSchema,
int fakeTotalCountOfSubtasks,
int fakeIndexOfThisSubtask,
AtomicReference<Throwable> thrownErrorUnderTest,
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
KinesisProxyInterface fakeKinesis) {
super(
fakeStreams,
sourceContext,
sourceContext.getCheckpointLock(),
getMockedRuntimeContext(fakeTotalCountOfSubtasks, fakeIndexOfThisSubtask),
fakeConfiguration,
deserializationSchema,
DEFAULT_SHARD_ASSIGNER,
null,
null,
thrownErrorUnderTest,
subscribedShardsStateUnderTest,
subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
(properties) -> fakeKinesis);
this.runWaiter = new OneShotLatch();
this.initialDiscoveryWaiter = new OneShotLatch();
this.shutdownWaiter = new OneShotLatch();
this.running = true;
}
Example #13
| Source File: FakeKinesisBehavioursFactory.java From flink with Apache License 2.0 | 5 votes |
|
public static KinesisProxyInterface initialNumOfRecordsAfterNumOfGetRecordsCallsWithAdaptiveReads(
final int numOfRecords,
final int numOfGetRecordsCalls,
final long millisBehindLatest) {
return new SingleShardEmittingAdaptiveNumOfRecordsKinesis(numOfRecords, numOfGetRecordsCalls,
millisBehindLatest);
}
Example #14
| Source File: FakeKinesisBehavioursFactory.java From flink with Apache License 2.0 | 5 votes |
|
public static KinesisProxyInterface totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator(
final int numOfRecords,
final int numOfGetRecordsCall,
final int orderOfCallToExpire,
final long millisBehindLatest) {
return new SingleShardEmittingFixNumOfRecordsWithExpiredIteratorKinesis(
numOfRecords, numOfGetRecordsCall, orderOfCallToExpire, millisBehindLatest);
}
Example #15
| Source File: FakeKinesisBehavioursFactory.java From flink with Apache License 2.0 | 5 votes |
|
public static KinesisProxyInterface noShardsFoundForRequestedStreamsBehaviour() {
return new KinesisProxyInterface() {
@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
return new GetShardListResult(); // not setting any retrieved shards for result
}
@Override
public String getShardIterator(StreamShardHandle shard, String shardIteratorType, Object startingMarker) {
return null;
}
@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
return null;
}
};
}
Example #16
| Source File: ShardConsumerTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedState() {
StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
subscribedShardsStateUnderTest.add(
new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));
TestSourceContext<String> sourceContext = new TestSourceContext<>();
TestableKinesisDataFetcher<String> fetcher =
new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
sourceContext,
new Properties(),
new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
new ShardConsumer<>(
fetcher,
shardIndex,
subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, 500L),
new ShardMetricsReporter()).run();
assertEquals(1000, sourceContext.getCollectedOutputs().size());
assertEquals(
SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
Example #17
| Source File: ShardConsumerTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testMetricsReporting() {
StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
subscribedShardsStateUnderTest.add(
new KinesisStreamShardState(
KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
fakeToBeConsumedShard,
new SequenceNumber("fakeStartingState")));
TestSourceContext<String> sourceContext = new TestSourceContext<>();
TestableKinesisDataFetcher<String> fetcher =
new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
sourceContext,
new Properties(),
new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
ShardMetricsReporter shardMetricsReporter = new ShardMetricsReporter();
long millisBehindLatest = 500L;
new ShardConsumer<>(
fetcher,
0,
subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, millisBehindLatest),
shardMetricsReporter).run();
// the millisBehindLatest metric should have been reported
assertEquals(millisBehindLatest, shardMetricsReporter.getMillisBehindLatest());
}
Example #18
| Source File: ShardConsumerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedState() {
StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
subscribedShardsStateUnderTest.add(
new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));
TestSourceContext<String> sourceContext = new TestSourceContext<>();
TestableKinesisDataFetcher<String> fetcher =
new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
sourceContext,
new Properties(),
new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
new ShardConsumer<>(
fetcher,
shardIndex,
subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, 500L),
new ShardMetricsReporter()).run();
assertEquals(1000, sourceContext.getCollectedOutputs().size());
assertEquals(
SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
Example #19
| Source File: TestableKinesisDataFetcherForShardConsumerException.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
public TestableKinesisDataFetcherForShardConsumerException(final List<String> fakeStreams,
final SourceFunction.SourceContext<T> sourceContext,
final Properties fakeConfiguration,
final KinesisDeserializationSchema<T> deserializationSchema,
final int fakeTotalCountOfSubtasks,
final int fakeIndexOfThisSubtask,
final AtomicReference<Throwable> thrownErrorUnderTest,
final LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
final HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
final KinesisProxyInterface fakeKinesis) {
super(fakeStreams, sourceContext, fakeConfiguration, deserializationSchema, fakeTotalCountOfSubtasks,
fakeIndexOfThisSubtask, thrownErrorUnderTest, subscribedShardsStateUnderTest,
subscribedStreamsToLastDiscoveredShardIdsStateUnderTest, fakeKinesis);
}
Example #20
| Source File: TestableKinesisDataFetcher.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
public TestableKinesisDataFetcher(
List<String> fakeStreams,
SourceFunction.SourceContext<T> sourceContext,
Properties fakeConfiguration,
KinesisDeserializationSchema<T> deserializationSchema,
int fakeTotalCountOfSubtasks,
int fakeIndexOfThisSubtask,
AtomicReference<Throwable> thrownErrorUnderTest,
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
KinesisProxyInterface fakeKinesis) {
super(
fakeStreams,
sourceContext,
sourceContext.getCheckpointLock(),
getMockedRuntimeContext(fakeTotalCountOfSubtasks, fakeIndexOfThisSubtask),
fakeConfiguration,
deserializationSchema,
DEFAULT_SHARD_ASSIGNER,
null,
null,
thrownErrorUnderTest,
subscribedShardsStateUnderTest,
subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
(properties) -> fakeKinesis);
this.runWaiter = new OneShotLatch();
this.initialDiscoveryWaiter = new OneShotLatch();
this.shutdownWaiter = new OneShotLatch();
this.running = true;
}
Example #21
| Source File: FakeKinesisBehavioursFactory.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
public static KinesisProxyInterface initialNumOfRecordsAfterNumOfGetRecordsCallsWithAdaptiveReads(
final int numOfRecords,
final int numOfGetRecordsCalls,
final long millisBehindLatest) {
return new SingleShardEmittingAdaptiveNumOfRecordsKinesis(numOfRecords, numOfGetRecordsCalls,
millisBehindLatest);
}
Example #22
| Source File: FakeKinesisBehavioursFactory.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
public static KinesisProxyInterface totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator(
final int numOfRecords,
final int numOfGetRecordsCall,
final int orderOfCallToExpire,
final long millisBehindLatest) {
return new SingleShardEmittingFixNumOfRecordsWithExpiredIteratorKinesis(
numOfRecords, numOfGetRecordsCall, orderOfCallToExpire, millisBehindLatest);
}
Example #23
| Source File: ShardConsumerTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithUnexpectedExpiredIterator() {
StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
subscribedShardsStateUnderTest.add(
new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));
TestSourceContext<String> sourceContext = new TestSourceContext<>();
TestableKinesisDataFetcher<String> fetcher =
new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
sourceContext,
new Properties(),
new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
new ShardConsumer<>(
fetcher,
shardIndex,
subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
// Get a total of 1000 records with 9 getRecords() calls,
// and the 7th getRecords() call will encounter an unexpected expired shard iterator
FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator(
1000, 9, 7, 500L),
new ShardMetricsReporter()).run();
assertEquals(1000, sourceContext.getCollectedOutputs().size());
assertEquals(
SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
Example #24
| Source File: ShardConsumerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithUnexpectedExpiredIterator() {
StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
subscribedShardsStateUnderTest.add(
new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));
TestSourceContext<String> sourceContext = new TestSourceContext<>();
TestableKinesisDataFetcher<String> fetcher =
new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
sourceContext,
new Properties(),
new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
new ShardConsumer<>(
fetcher,
shardIndex,
subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
// Get a total of 1000 records with 9 getRecords() calls,
// and the 7th getRecords() call will encounter an unexpected expired shard iterator
FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator(
1000, 9, 7, 500L),
new ShardMetricsReporter()).run();
assertEquals(1000, sourceContext.getCollectedOutputs().size());
assertEquals(
SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
Example #25
| Source File: FakeKinesisBehavioursFactory.java From flink with Apache License 2.0 | 4 votes |
|
public static KinesisProxyInterface blockingQueueGetRecords(Map<String, List<BlockingQueue<String>>> streamsToShardQueues) {
return new BlockingQueueKinesis(streamsToShardQueues);
}
Example #26
| Source File: ShardConsumerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithAdaptiveReads() {
Properties consumerProperties = new Properties();
consumerProperties.put("flink.shard.adaptivereads", "true");
StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
subscribedShardsStateUnderTest.add(
new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));
TestSourceContext<String> sourceContext = new TestSourceContext<>();
TestableKinesisDataFetcher<String> fetcher =
new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
sourceContext,
consumerProperties,
new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
new ShardConsumer<>(
fetcher,
shardIndex,
subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
// Initial number of records to fetch --> 10
FakeKinesisBehavioursFactory.initialNumOfRecordsAfterNumOfGetRecordsCallsWithAdaptiveReads(10, 2, 500L),
new ShardMetricsReporter()).run();
// Avg record size for first batch --> 10 * 10 Kb/10 = 10 Kb
// Number of records fetched in second batch --> 2 Mb/10Kb * 5 = 40
// Total number of records = 10 + 40 = 50
assertEquals(50, sourceContext.getCollectedOutputs().size());
assertEquals(
SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
Example #27
| Source File: FakeKinesisBehavioursFactory.java From flink with Apache License 2.0 | 4 votes |
|
public static KinesisProxyInterface totalNumOfRecordsAfterNumOfGetRecordsCalls(
final int numOfRecords,
final int numOfGetRecordsCalls,
final long millisBehindLatest) {
return new SingleShardEmittingFixNumOfRecordsKinesis(numOfRecords, numOfGetRecordsCalls, millisBehindLatest);
}
Example #28
| Source File: FakeKinesisBehavioursFactory.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
public static KinesisProxyInterface totalNumOfRecordsAfterNumOfGetRecordsCalls(
final int numOfRecords,
final int numOfGetRecordsCalls,
final long millisBehindLatest) {
return new SingleShardEmittingFixNumOfRecordsKinesis(numOfRecords, numOfGetRecordsCalls, millisBehindLatest);
}
Example #29
| Source File: ShardConsumerTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithAdaptiveReads() {
Properties consumerProperties = new Properties();
consumerProperties.put("flink.shard.adaptivereads", "true");
StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
subscribedShardsStateUnderTest.add(
new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));
TestSourceContext<String> sourceContext = new TestSourceContext<>();
KinesisDeserializationSchemaWrapper<String> deserializationSchema = new KinesisDeserializationSchemaWrapper<>(
new SimpleStringSchema());
TestableKinesisDataFetcher<String> fetcher =
new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
sourceContext,
consumerProperties,
deserializationSchema,
10,
2,
new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
new ShardConsumer<>(
fetcher,
shardIndex,
subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
// Initial number of records to fetch --> 10
FakeKinesisBehavioursFactory.initialNumOfRecordsAfterNumOfGetRecordsCallsWithAdaptiveReads(10, 2, 500L),
new ShardMetricsReporter(),
deserializationSchema)
.run();
// Avg record size for first batch --> 10 * 10 Kb/10 = 10 Kb
// Number of records fetched in second batch --> 2 Mb/10Kb * 5 = 40
// Total number of records = 10 + 40 = 50
assertEquals(50, sourceContext.getCollectedOutputs().size());
assertEquals(
SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
Example #30
| Source File: ShardConsumerTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithUnexpectedExpiredIterator() {
StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
subscribedShardsStateUnderTest.add(
new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));
TestSourceContext<String> sourceContext = new TestSourceContext<>();
KinesisDeserializationSchemaWrapper<String> deserializationSchema = new KinesisDeserializationSchemaWrapper<>(
new SimpleStringSchema());
TestableKinesisDataFetcher<String> fetcher =
new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
sourceContext,
new Properties(),
deserializationSchema,
10,
2,
new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
new ShardConsumer<>(
fetcher,
shardIndex,
subscribedShardsStateUnderTest.get(0).getStreamShardHandle(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(),
// Get a total of 1000 records with 9 getRecords() calls,
// and the 7th getRecords() call will encounter an unexpected expired shard iterator
FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator(
1000, 9, 7, 500L),
new ShardMetricsReporter(),
deserializationSchema)
.run();
assertEquals(1000, sourceContext.getCollectedOutputs().size());
assertEquals(
SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
