software.amazon.awssdk.services.kinesis.model.GetShardIteratorResponse Java Examples

@Test
public void testGetRecordsThrowsSdkException() throws Exception {
    expectedExceptionRule.expect(SdkException.class);
    expectedExceptionRule.expectMessage("Test Exception");

    CompletableFuture<GetShardIteratorResponse> getShardIteratorFuture = CompletableFuture
            .completedFuture(GetShardIteratorResponse.builder().shardIterator("test").build());

    // Set up proxy mock methods
    when(kinesisClient.getShardIterator(any(GetShardIteratorRequest.class))).thenReturn(getShardIteratorFuture);
    when(kinesisClient.getRecords(any(GetRecordsRequest.class))).thenReturn(getRecordsResponseFuture);
    when(getRecordsResponseFuture.get(anyLong(), any(TimeUnit.class)))
            .thenThrow(new ExecutionException(SdkException.builder().message("Test Exception").build()));

    // Create data fectcher and initialize it with latest type checkpoint
    kinesisDataFetcher.initialize(SentinelCheckpoint.LATEST.toString(), INITIAL_POSITION_LATEST);
    final GetRecordsRetrievalStrategy getRecordsRetrievalStrategy = new SynchronousGetRecordsRetrievalStrategy(
            kinesisDataFetcher);

    // Call records of dataFetcher which will throw an exception
    getRecordsRetrievalStrategy.getRecords(MAX_RECORDS);

}
 

@Test
public void testTimeoutExceptionIsRetryableForGetRecords() throws Exception {
    expectedExceptionRule.expect(RetryableRetrievalException.class);
    expectedExceptionRule.expectCause(isA(TimeoutException.class));
    expectedExceptionRule.expectMessage("Timeout");

    CompletableFuture<GetShardIteratorResponse> getShardIteratorFuture = CompletableFuture
            .completedFuture(GetShardIteratorResponse.builder().shardIterator("test").build());

    // Set up proxy mock methods
    when(kinesisClient.getShardIterator(any(GetShardIteratorRequest.class))).thenReturn(getShardIteratorFuture);
    when(kinesisClient.getRecords(any(GetRecordsRequest.class))).thenReturn(getRecordsResponseFuture);
    when(getRecordsResponseFuture.get(anyLong(), any(TimeUnit.class))).thenThrow(new TimeoutException("Timeout"));

    // Create data fectcher and initialize it with latest type checkpoint
    kinesisDataFetcher.initialize(SentinelCheckpoint.LATEST.toString(), INITIAL_POSITION_LATEST);
    final GetRecordsRetrievalStrategy getRecordsRetrievalStrategy = new SynchronousGetRecordsRetrievalStrategy(
            kinesisDataFetcher);

    // Call records of dataFetcher which will throw an exception
    getRecordsRetrievalStrategy.getRecords(MAX_RECORDS);
}
 

@Before
public void setup() throws Exception {
    records = new ArrayList<>();
    dataFetcher = spy(new KinesisDataFetcherForTest(kinesisClient, streamName, shardId, MAX_RECORDS_PER_CALL));
    getRecordsRetrievalStrategy = Mockito.spy(new SynchronousGetRecordsRetrievalStrategy(dataFetcher));
    executorService = spy(Executors.newFixedThreadPool(1));
    CompletableFuture<GetShardIteratorResponse> future = mock(CompletableFuture.class);

    when(extendedSequenceNumber.sequenceNumber()).thenReturn("LATEST");
    when(future.get(anyLong(), any(TimeUnit.class))).thenReturn(GetShardIteratorResponse.builder().shardIterator("TestIterator").build());
    when(kinesisClient.getShardIterator(any(GetShardIteratorRequest.class))).thenReturn(future);

    getRecordsCache = new PrefetchRecordsPublisher(MAX_SIZE,
            MAX_BYTE_SIZE,
            MAX_RECORDS_COUNT,
            MAX_RECORDS_PER_CALL,
            getRecordsRetrievalStrategy,
            executorService,
            IDLE_MILLIS_BETWEEN_CALLS,
            new NullMetricsFactory(),
            operation,
            "test-shard");
}
 

private void testGets(final String streamName, final Shard shard) {
    // Get an iterator for the first shard.
    GetShardIteratorResponse iteratorResult = client.getShardIterator(
            GetShardIteratorRequest.builder()
                                   .streamName(streamName)
                                   .shardId(shard.shardId())
                                   .shardIteratorType(ShardIteratorType.AT_SEQUENCE_NUMBER)
                                   .startingSequenceNumber(shard.sequenceNumberRange().startingSequenceNumber())
                                   .build());
    Assert.assertNotNull(iteratorResult);

    String iterator = iteratorResult.shardIterator();
    Assert.assertNotNull(iterator);

    int tries = 0;
    GetRecordsResponse result;
    List<Record> records;

    // Read the first record from the first shard (looping until it's
    // available).
    while (true) {
        tries += 1;
        if (tries > 100) {
            Assert.fail("Failed to read any records after 100 seconds");
        }

        result = client.getRecords(GetRecordsRequest.builder()
                                                    .shardIterator(iterator)
                                                    .limit(1)
                                                    .build());
        Assert.assertNotNull(result);
        Assert.assertNotNull(result.records());
        Assert.assertNotNull(result.nextShardIterator());

        records = result.records();
        if (records.size() > 0) {
            long arrivalTime = records.get(0).approximateArrivalTimestamp().toEpochMilli();
            Long delta = Math.abs(Instant.now().minusMillis(arrivalTime).toEpochMilli());
            // Assert that the arrival date is within 5 minutes of the current date to make sure it unmarshalled correctly.
            assertThat(delta, Matchers.lessThan(60 * 5000L));
            break;
        }

        try {
            Thread.sleep(1000);
        } catch (InterruptedException exception) {
            throw new RuntimeException(exception);
        }

        iterator = result.nextShardIterator();
    }

    System.out.println("  [Succeeded after " + tries + " tries]");
    Assert.assertEquals(1, records.size());
    validateRecord(records.get(0), "See No Evil");

    // Read the second record from the first shard.
    result = client.getRecords(GetRecordsRequest.builder()
                                                .shardIterator(result.nextShardIterator())
                                                .build());
    Assert.assertNotNull(result);
    Assert.assertNotNull(result.records());
    Assert.assertNotNull(result.nextShardIterator());

    records = result.records();
    Assert.assertEquals(1, records.size());
    validateRecord(records.get(0), "See No Evil");

    // Try to read some more, get EOF.
    result = client.getRecords(GetRecordsRequest.builder()
                                                .shardIterator(result.nextShardIterator())
                                                .build());
    Assert.assertNotNull(result);
    Assert.assertNotNull(result.records());
    Assert.assertTrue(result.records().isEmpty());
    Assert.assertNull(result.nextShardIterator());
}
 

public static void getStockTrades(KinesisClient kinesisClient, String streamName) {

            String shardIterator;
            String lastShardId = null;

            // Retrieve the shards from a data stream
            DescribeStreamRequest describeStreamRequest = DescribeStreamRequest.builder()
                    .streamName(streamName)
                    .build();
            List<Shard> shards = new ArrayList<>();

            DescribeStreamResponse streamRes;
            do {
                streamRes = kinesisClient.describeStream(describeStreamRequest);
                shards.addAll(streamRes.streamDescription().shards());

                if (shards.size() > 0) {
                    lastShardId = shards.get(shards.size() - 1).shardId();
                }
            } while (streamRes.streamDescription().hasMoreShards());

            GetShardIteratorRequest itReq = GetShardIteratorRequest.builder()
                    .streamName(streamName)
                    .shardIteratorType("TRIM_HORIZON")
                    .shardId(shards.get(0).shardId())
                    .build();

            GetShardIteratorResponse shardIteratorResult = kinesisClient.getShardIterator(itReq);
            shardIterator = shardIteratorResult.shardIterator();

            // Continuously read data records from a shard
            List<Record> records;

            // Create a GetRecordsRequest with the existing shardIterator,
            // and set maximum records to return to 1000
            GetRecordsRequest recordsRequest = GetRecordsRequest.builder()
                     .shardIterator(shardIterator)
                     .limit(1000)
                     .build();

           GetRecordsResponse result = kinesisClient.getRecords(recordsRequest);

           // Put result into a record list, result might be empty
           records = result.records();

            // Print records
            for (Record record : records) {
                SdkBytes byteBuffer = record.data();
                System.out.println(String.format("Seq No: %s - %s", record.sequenceNumber(),
                 new String(byteBuffer.asByteArray())));
             }

             }
 

private CompletableFuture<GetShardIteratorResponse> makeGetShardIteratorResonse(String shardIterator)
        throws InterruptedException, ExecutionException {
    return CompletableFuture
            .completedFuture(GetShardIteratorResponse.builder().shardIterator(shardIterator).build());
}