Java Code Examples for com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream#TRIM_HORIZON

The following examples show how to use com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream#TRIM_HORIZON . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ShardRecordsIterator.java    From beam with Apache License 2.0 6 votes vote down vote up 
List<ShardRecordsIterator> findSuccessiveShardRecordIterators() throws TransientKinesisException {
  List<Shard> shards = kinesis.listShards(streamName);
  List<ShardRecordsIterator> successiveShardRecordIterators = new ArrayList<>();
  for (Shard shard : shards) {
    if (shardId.equals(shard.getParentShardId())) {
      ShardCheckpoint shardCheckpoint =
          new ShardCheckpoint(
              streamName,
              shard.getShardId(),
              new StartingPoint(InitialPositionInStream.TRIM_HORIZON));
      successiveShardRecordIterators.add(
          new ShardRecordsIterator(shardCheckpoint, kinesis, watermarkPolicyFactory));
    }
  }
  return successiveShardRecordIterators;
}
Example 2
Source File: KinesisSourceIT.java    From datacollector with Apache License 2.0 6 votes vote down vote up 
private KinesisConsumerConfigBean getKinesisConsumerConfig(String streamName) {
  KinesisConsumerConfigBean conf = new KinesisConsumerConfigBean();
  conf.dataFormatConfig = new DataParserFormatConfig();
  conf.awsConfig = new AWSConfig();
  conf.awsConfig.awsAccessKeyId = () -> "foo";
  conf.awsConfig.awsSecretAccessKey = () -> "boo";

  conf.region = AwsRegion.OTHER;
  conf.endpoint = getKinesisEndpoint();
  conf.streamName = streamName;

  conf.dataFormat = DataFormat.JSON;
  conf.dataFormatConfig.jsonContent = JsonMode.MULTIPLE_OBJECTS;
  conf.dataFormatConfig.charset = "UTF-8";
  conf.dataFormatConfig.jsonMaxObjectLen = 1024;

  conf.applicationName = UUID.randomUUID().toString();
  conf.idleTimeBetweenReads = 250;
  conf.initialPositionInStream = InitialPositionInStream.TRIM_HORIZON;
  conf.maxBatchSize = 1000;
  conf.maxRecordProcessors = 2; // Must be at least 1

  return conf;
}
Example 3
Source File: StartingPointShardsFinderTest.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Test
public void shouldFindEarliestShardsWhenTrimHorizonStartingPointRequested() throws Exception {
  // given
  StartingPoint trimHorizonStartingPoint =
      new StartingPoint(InitialPositionInStream.TRIM_HORIZON);
  when(kinesis.listShards(STREAM_NAME)).thenReturn(allShards);

  // when
  Iterable<Shard> shardsAtStartingPoint =
      underTest.findShardsAtStartingPoint(kinesis, STREAM_NAME, trimHorizonStartingPoint);

  // then
  assertThat(shardsAtStartingPoint).containsExactlyInAnyOrder(shard00, shard01);
}
Example 4
Source File: KinesisSource.java    From datacollector with Apache License 2.0 5 votes vote down vote up 
private Worker createKinesisWorker(IRecordProcessorFactory recordProcessorFactory, int maxBatchSize) {
  KinesisClientLibConfiguration kclConfig =
      new KinesisClientLibConfiguration(
          conf.applicationName,
          conf.streamName,
          credentials,
          getWorkerId()
      );

  kclConfig
      .withMaxRecords(maxBatchSize)
      .withCallProcessRecordsEvenForEmptyRecordList(false)
      .withIdleTimeBetweenReadsInMillis(conf.idleTimeBetweenReads)
      .withKinesisClientConfig(clientConfiguration);

  if (conf.initialPositionInStream == InitialPositionInStream.AT_TIMESTAMP) {
    kclConfig.withTimestampAtInitialPositionInStream(new Date(conf.initialTimestamp));
  } else if (conf.initialPositionInStream == InitialPositionInStream.LATEST || conf.initialPositionInStream == InitialPositionInStream.TRIM_HORIZON) {
    kclConfig.withInitialPositionInStream(conf.initialPositionInStream);
  }

  if (conf.region == AwsRegion.OTHER) {
    kclConfig.withKinesisEndpoint(conf.endpoint);
  } else {
    kclConfig.withRegionName(conf.region.getId());
  }

  return new Worker.Builder()
      .recordProcessorFactory(recordProcessorFactory)
      .metricsFactory(metricsFactory)
      .dynamoDBClient(dynamoDBClient)
      .cloudWatchClient(cloudWatchClient)
      .execService(executor)
      .config(kclConfig)
      .build();
}
Example 5
Source File: KinesisInputRuntime.java    From components with Apache License 2.0 5 votes vote down vote up 
private InitialPositionInStream convertToPosition(KinesisInputProperties.OffsetType offsetType) {
    switch (offsetType) {
    case LATEST:
        return InitialPositionInStream.LATEST;
    case EARLIEST:
        return InitialPositionInStream.TRIM_HORIZON;
    default:
        TalendRuntimeException.build(CommonErrorCodes.UNEXPECTED_ARGUMENT).setAndThrow(
                String.format("Do not support OffsetType %s", offsetType));
        return null;
    }
}