Java Code Examples for com.amazonaws.services.kinesis.model.Shard#getShardId()

The following examples show how to use com.amazonaws.services.kinesis.model.Shard#getShardId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: KinesisSplitManager.java From presto with Apache License 2.0

6 votes

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableHandle table, ConnectorSplitManager.SplitSchedulingStrategy splitSchedulingStrategy)
{
    KinesisTableHandle kinesisTableHandle = (KinesisTableHandle) table;

    InternalStreamDescription description = this.getStreamDescription(kinesisTableHandle.getStreamName());

    ImmutableList.Builder<ConnectorSplit> builder = ImmutableList.builder();
    for (Shard shard : description.getShards()) {
        KinesisSplit split = new KinesisSplit(
                kinesisTableHandle.getStreamName(),
                kinesisTableHandle.getMessageDataFormat(),
                shard.getShardId(),
                shard.getSequenceNumberRange().getStartingSequenceNumber(),
                shard.getSequenceNumberRange().getEndingSequenceNumber());
        builder.add(split);
    }

    return new FixedSplitSource(builder.build());
}

Example 2

Source File: ShardRecordsIterator.java From beam with Apache License 2.0

6 votes

List<ShardRecordsIterator> findSuccessiveShardRecordIterators() throws TransientKinesisException {
  List<Shard> shards = kinesis.listShards(streamName);
  List<ShardRecordsIterator> successiveShardRecordIterators = new ArrayList<>();
  for (Shard shard : shards) {
    if (shardId.equals(shard.getParentShardId())) {
      ShardCheckpoint shardCheckpoint =
          new ShardCheckpoint(
              streamName,
              shard.getShardId(),
              new StartingPoint(InitialPositionInStream.TRIM_HORIZON));
      successiveShardRecordIterators.add(
          new ShardRecordsIterator(shardCheckpoint, kinesis, watermarkPolicyFactory));
    }
  }
  return successiveShardRecordIterators;
}

Example 3

Source File: KinesisSplitManager.java From presto-kinesis with Apache License 2.0

6 votes

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout, ConnectorSplitManager.SplitSchedulingStrategy splitSchedulingStrategy)
{
    KinesisTableLayoutHandle kinesislayout = handleResolver.convertLayout(layout);
    KinesisTableHandle kinesisTableHandle = kinesislayout.getTable();

    InternalStreamDescription desc = this.getStreamDescription(kinesisTableHandle.getStreamName());

    ImmutableList.Builder<ConnectorSplit> builder = ImmutableList.builder();
    for (Shard shard : desc.getShards()) {
        KinesisSplit split = new KinesisSplit(connectorId,
                kinesisTableHandle.getStreamName(),
                kinesisTableHandle.getMessageDataFormat(),
                shard.getShardId(),
                shard.getSequenceNumberRange().getStartingSequenceNumber(),
                shard.getSequenceNumberRange().getEndingSequenceNumber());
        builder.add(split);
    }

    return new FixedSplitSource(builder.build());
}

Example 4

Source File: StartingPointShardsFinderTest.java From beam with Apache License 2.0

5 votes

private void prepareShard(
    Shard shard,
    String nextIterator,
    ShardIteratorType shardIteratorType,
    Instant startTimestamp) {
  try {
    String shardIterator = shardIteratorType + shard.getShardId() + "-current";
    if (shardIteratorType == ShardIteratorType.AT_TIMESTAMP) {
      when(kinesis.getShardIterator(
              STREAM_NAME,
              shard.getShardId(),
              ShardIteratorType.AT_TIMESTAMP,
              null,
              startTimestamp))
          .thenReturn(shardIterator);
    } else {
      when(kinesis.getShardIterator(
              STREAM_NAME, shard.getShardId(), shardIteratorType, null, null))
          .thenReturn(shardIterator);
    }
    GetKinesisRecordsResult result =
        new GetKinesisRecordsResult(
            Collections.<UserRecord>emptyList(),
            nextIterator,
            0,
            STREAM_NAME,
            shard.getShardId());
    when(kinesis.getRecords(shardIterator, STREAM_NAME, shard.getShardId())).thenReturn(result);
  } catch (TransientKinesisException e) {
    throw new RuntimeException(e);
  }
}