com.amazonaws.services.kinesis.model.GetShardIteratorRequest Java Examples

The following examples show how to use com.amazonaws.services.kinesis.model.GetShardIteratorRequest. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SimplifiedKinesisClientTest.java    From beam with Apache License 2.0 6 votes vote down vote up 
@Test
public void shouldReturnIteratorStartingWithTimestamp() throws Exception {
  Instant timestamp = Instant.now();
  when(kinesis.getShardIterator(
          new GetShardIteratorRequest()
              .withStreamName(STREAM)
              .withShardId(SHARD_1)
              .withShardIteratorType(ShardIteratorType.AT_SEQUENCE_NUMBER)
              .withTimestamp(timestamp.toDate())))
      .thenReturn(new GetShardIteratorResult().withShardIterator(SHARD_ITERATOR));

  String stream =
      underTest.getShardIterator(
          STREAM, SHARD_1, ShardIteratorType.AT_SEQUENCE_NUMBER, null, timestamp);

  assertThat(stream).isEqualTo(SHARD_ITERATOR);
}
Example #2
Source File: KinesisProxy.java    From flink with Apache License 2.0 6 votes vote down vote up 
private String getShardIterator(GetShardIteratorRequest getShardIteratorRequest) throws InterruptedException {
	GetShardIteratorResult getShardIteratorResult = null;

	int retryCount = 0;
	while (retryCount <= getShardIteratorMaxRetries && getShardIteratorResult == null) {
		try {
				getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
		} catch (AmazonServiceException ex) {
			if (isRecoverableException(ex)) {
				long backoffMillis = fullJitterBackoff(
					getShardIteratorBaseBackoffMillis, getShardIteratorMaxBackoffMillis, getShardIteratorExpConstant, retryCount++);
				LOG.warn("Got recoverable AmazonServiceException. Backing off for "
					+ backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
				Thread.sleep(backoffMillis);
			} else {
				throw ex;
			}
		}
	}

	if (getShardIteratorResult == null) {
		throw new RuntimeException("Retries exceeded for getShardIterator operation - all " + getShardIteratorMaxRetries +
			" retry attempts failed.");
	}
	return getShardIteratorResult.getShardIterator();
}
Example #3
Source File: KinesisUtil.java    From datacollector with Apache License 2.0 6 votes vote down vote up 
public static List<com.amazonaws.services.kinesis.model.Record> getPreviewRecords(
    ClientConfiguration awsClientConfig,
    KinesisConfigBean conf,
    int maxBatchSize,
    GetShardIteratorRequest getShardIteratorRequest
) throws StageException {
  AmazonKinesis kinesisClient = getKinesisClient(awsClientConfig, conf);

  GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
  String shardIterator = getShardIteratorResult.getShardIterator();

  GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
  getRecordsRequest.setShardIterator(shardIterator);
  getRecordsRequest.setLimit(maxBatchSize);

  GetRecordsResult getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
  return getRecordsResult.getRecords();
}
Example #4
Source File: SimplifiedKinesisClientTest.java    From beam with Apache License 2.0 6 votes vote down vote up 
private void shouldHandleGetShardIteratorError(
    Exception thrownException, Class<? extends Exception> expectedExceptionClass) {
  GetShardIteratorRequest request =
      new GetShardIteratorRequest()
          .withStreamName(STREAM)
          .withShardId(SHARD_1)
          .withShardIteratorType(ShardIteratorType.LATEST);

  when(kinesis.getShardIterator(request)).thenThrow(thrownException);

  try {
    underTest.getShardIterator(STREAM, SHARD_1, ShardIteratorType.LATEST, null, null);
    failBecauseExceptionWasNotThrown(expectedExceptionClass);
  } catch (Exception e) {
    assertThat(e).isExactlyInstanceOf(expectedExceptionClass);
  } finally {
    reset(kinesis);
  }
}
Example #5
Source File: SimplifiedKinesisClientTest.java    From beam with Apache License 2.0 6 votes vote down vote up 
@Test
public void shouldReturnIteratorStartingWithSequenceNumber() throws Exception {
  when(kinesis.getShardIterator(
          new GetShardIteratorRequest()
              .withStreamName(STREAM)
              .withShardId(SHARD_1)
              .withShardIteratorType(ShardIteratorType.AT_SEQUENCE_NUMBER)
              .withStartingSequenceNumber(SEQUENCE_NUMBER)))
      .thenReturn(new GetShardIteratorResult().withShardIterator(SHARD_ITERATOR));

  String stream =
      underTest.getShardIterator(
          STREAM, SHARD_1, ShardIteratorType.AT_SEQUENCE_NUMBER, SEQUENCE_NUMBER, null);

  assertThat(stream).isEqualTo(SHARD_ITERATOR);
}
Example #6
Source File: SimplifiedKinesisClient.java    From beam with Apache License 2.0 6 votes vote down vote up 
public String getShardIterator(
    final String streamName,
    final String shardId,
    final ShardIteratorType shardIteratorType,
    final String startingSequenceNumber,
    final Instant timestamp)
    throws TransientKinesisException {
  final Date date = timestamp != null ? timestamp.toDate() : null;
  return wrapExceptions(
      () ->
          kinesis
              .getShardIterator(
                  new GetShardIteratorRequest()
                      .withStreamName(streamName)
                      .withShardId(shardId)
                      .withShardIteratorType(shardIteratorType)
                      .withStartingSequenceNumber(startingSequenceNumber)
                      .withTimestamp(date))
              .getShardIterator());
}
Example #7
Source File: KinesisProxy.java    From flink with Apache License 2.0 6 votes vote down vote up 
private String getShardIterator(GetShardIteratorRequest getShardIteratorRequest) throws InterruptedException {
	GetShardIteratorResult getShardIteratorResult = null;

	int retryCount = 0;
	while (retryCount <= getShardIteratorMaxRetries && getShardIteratorResult == null) {
		try {
				getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
		} catch (AmazonServiceException ex) {
			if (isRecoverableException(ex)) {
				long backoffMillis = fullJitterBackoff(
					getShardIteratorBaseBackoffMillis, getShardIteratorMaxBackoffMillis, getShardIteratorExpConstant, retryCount++);
				LOG.warn("Got recoverable AmazonServiceException. Backing off for "
					+ backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
				Thread.sleep(backoffMillis);
			} else {
				throw ex;
			}
		}
	}

	if (getShardIteratorResult == null) {
		throw new RuntimeException("Retries exceeded for getShardIterator operation - all " + getShardIteratorMaxRetries +
			" retry attempts failed.");
	}
	return getShardIteratorResult.getShardIterator();
}
Example #8
Source File: KinesisProxy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private String getShardIterator(GetShardIteratorRequest getShardIteratorRequest) throws InterruptedException {
	GetShardIteratorResult getShardIteratorResult = null;

	int retryCount = 0;
	while (retryCount <= getShardIteratorMaxRetries && getShardIteratorResult == null) {
		try {
				getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
		} catch (AmazonServiceException ex) {
			if (isRecoverableException(ex)) {
				long backoffMillis = fullJitterBackoff(
					getShardIteratorBaseBackoffMillis, getShardIteratorMaxBackoffMillis, getShardIteratorExpConstant, retryCount++);
				LOG.warn("Got recoverable AmazonServiceException. Backing off for "
					+ backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
				Thread.sleep(backoffMillis);
			} else {
				throw ex;
			}
		}
	}

	if (getShardIteratorResult == null) {
		throw new RuntimeException("Retries exceeded for getShardIterator operation - all " + getShardIteratorMaxRetries +
			" retry attempts failed.");
	}
	return getShardIteratorResult.getShardIterator();
}
Example #9
Source File: MockKinesisClient.java    From presto with Apache License 2.0 6 votes vote down vote up 
@Override
public GetShardIteratorResult getShardIterator(GetShardIteratorRequest getShardIteratorRequest)
        throws AmazonClientException
{
    ShardIterator iter = ShardIterator.fromStreamAndShard(getShardIteratorRequest.getStreamName(), getShardIteratorRequest.getShardId());
    if (iter == null) {
        throw new AmazonClientException("Bad stream or shard iterator!");
    }

    InternalStream theStream = this.getStream(iter.streamId);
    if (theStream == null) {
        throw new AmazonClientException("Unknown stream or bad shard iterator!");
    }

    String seqAsString = getShardIteratorRequest.getStartingSequenceNumber();
    if (seqAsString != null && !seqAsString.isEmpty() && getShardIteratorRequest.getShardIteratorType().equals("AFTER_SEQUENCE_NUMBER")) {
        int sequence = parseInt(seqAsString);
        iter.recordIndex = sequence + 1;
    }
    else {
        iter.recordIndex = 100;
    }

    GetShardIteratorResult result = new GetShardIteratorResult();
    return result.withShardIterator(iter.makeString());
}
Example #10
Source File: KinesisProxy.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * {@inheritDoc}
 */
@Override
public String getShardIterator(StreamShardHandle shard, String shardIteratorType, @Nullable Object startingMarker) throws InterruptedException {
	GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
		.withStreamName(shard.getStreamName())
		.withShardId(shard.getShard().getShardId())
		.withShardIteratorType(shardIteratorType);

	switch (ShardIteratorType.fromValue(shardIteratorType)) {
		case TRIM_HORIZON:
		case LATEST:
			break;
		case AT_TIMESTAMP:
			if (startingMarker instanceof Date) {
				getShardIteratorRequest.setTimestamp((Date) startingMarker);
			} else {
				throw new IllegalArgumentException("Invalid object given for GetShardIteratorRequest() when ShardIteratorType is AT_TIMESTAMP. Must be a Date object.");
			}
			break;
		case AT_SEQUENCE_NUMBER:
		case AFTER_SEQUENCE_NUMBER:
			if (startingMarker instanceof String) {
				getShardIteratorRequest.setStartingSequenceNumber((String) startingMarker);
			} else {
				throw new IllegalArgumentException("Invalid object given for GetShardIteratorRequest() when ShardIteratorType is AT_SEQUENCE_NUMBER or AFTER_SEQUENCE_NUMBER. Must be a String.");
			}
	}
	return getShardIterator(getShardIteratorRequest);
}
Example #11
Source File: KinesisRecordSet.java    From presto with Apache License 2.0 5 votes vote down vote up 
private void getIterator()
        throws ResourceNotFoundException
{
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(split.getStreamName());
    getShardIteratorRequest.setShardId(split.getShardId());

    // Explanation: when we have a sequence number from a prior read or checkpoint, always use it.
    // Otherwise, decide if starting at a timestamp or the trim horizon based on configuration.
    // If starting at a timestamp, use the session variable STARTING_TIMESTAMP when given, otherwise
    // fallback on starting at STARTING_OFFSET_SECONDS from timestamp.
    if (lastReadSequenceNumber == null) {
        if (isIteratorFromTimestamp(session)) {
            getShardIteratorRequest.setShardIteratorType("AT_TIMESTAMP");
            long iteratorStartTimestamp = getIteratorStartTimestamp(session);
            if (iteratorStartTimestamp == 0) {
                long startTimestamp = System.currentTimeMillis() - (getIteratorOffsetSeconds(session) * 1000);
                getShardIteratorRequest.setTimestamp(new Date(startTimestamp));
            }
            else {
                getShardIteratorRequest.setTimestamp(new Date(iteratorStartTimestamp));
            }
        }
        else {
            getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON");
        }
    }
    else {
        getShardIteratorRequest.setShardIteratorType("AFTER_SEQUENCE_NUMBER");
        getShardIteratorRequest.setStartingSequenceNumber(lastReadSequenceNumber);
    }

    GetShardIteratorResult getShardIteratorResult = clientManager.getClient().getShardIterator(getShardIteratorRequest);
    shardIterator = getShardIteratorResult.getShardIterator();
}
Example #12
Source File: AmazonKinesisMock.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Override
public GetShardIteratorResult getShardIterator(GetShardIteratorRequest getShardIteratorRequest) {
  ShardIteratorType shardIteratorType =
      ShardIteratorType.fromValue(getShardIteratorRequest.getShardIteratorType());

  String shardIterator;
  if (shardIteratorType == ShardIteratorType.TRIM_HORIZON) {
    shardIterator = String.format("%s:%s", getShardIteratorRequest.getShardId(), 0);
  } else {
    throw new RuntimeException("Not implemented");
  }

  return new GetShardIteratorResult().withShardIterator(shardIterator);
}
Example #13
Source File: KinesisSource.java    From datacollector with Apache License 2.0 5 votes vote down vote up 
private void previewProcess(
    int maxBatchSize,
    BatchMaker batchMaker
) throws IOException, StageException {
  ClientConfiguration awsClientConfig = AWSUtil.getClientConfiguration(conf.proxyConfig);

  String shardId = KinesisUtil.getLastShardId(awsClientConfig, conf, conf.streamName);

  GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
  getShardIteratorRequest.setStreamName(conf.streamName);
  getShardIteratorRequest.setShardId(shardId);
  getShardIteratorRequest.setShardIteratorType(conf.initialPositionInStream.name());

  if (conf.initialPositionInStream == InitialPositionInStream.AT_TIMESTAMP) {
    getShardIteratorRequest.setTimestamp(new Date(conf.initialTimestamp));
  }

  if (!getContext().isPreview() && conf.maxBatchSize > maxBatchSize) {
    getContext().reportError(Errors.KINESIS_18, maxBatchSize);
  }

  List<com.amazonaws.services.kinesis.model.Record> results = KinesisUtil.getPreviewRecords(
      awsClientConfig,
      conf,
      Math.min(conf.maxBatchSize, maxBatchSize),
      getShardIteratorRequest
  );

  int batchSize = results.size() > maxBatchSize ? maxBatchSize : results.size();

  for (int index = 0; index < batchSize; index++) {
    com.amazonaws.services.kinesis.model.Record record = results.get(index);
    UserRecord userRecord = new UserRecord(record);
    KinesisUtil.processKinesisRecord(
        getShardIteratorRequest.getShardId(),
        userRecord,
        parserFactory
    ).forEach(batchMaker::addRecord);
  }
}
Example #14
Source File: KinesisProxy.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * {@inheritDoc}
 */
@Override
public String getShardIterator(StreamShardHandle shard, String shardIteratorType, @Nullable Object startingMarker) throws InterruptedException {
	GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
		.withStreamName(shard.getStreamName())
		.withShardId(shard.getShard().getShardId())
		.withShardIteratorType(shardIteratorType);

	switch (ShardIteratorType.fromValue(shardIteratorType)) {
		case TRIM_HORIZON:
		case LATEST:
			break;
		case AT_TIMESTAMP:
			if (startingMarker instanceof Date) {
				getShardIteratorRequest.setTimestamp((Date) startingMarker);
			} else {
				throw new IllegalArgumentException("Invalid object given for GetShardIteratorRequest() when ShardIteratorType is AT_TIMESTAMP. Must be a Date object.");
			}
			break;
		case AT_SEQUENCE_NUMBER:
		case AFTER_SEQUENCE_NUMBER:
			if (startingMarker instanceof String) {
				getShardIteratorRequest.setStartingSequenceNumber((String) startingMarker);
			} else {
				throw new IllegalArgumentException("Invalid object given for GetShardIteratorRequest() when ShardIteratorType is AT_SEQUENCE_NUMBER or AFTER_SEQUENCE_NUMBER. Must be a String.");
			}
	}
	return getShardIterator(getShardIteratorRequest);
}
Example #15
Source File: KinesisRecordSet.java    From presto-kinesis with Apache License 2.0 5 votes vote down vote up 
private void getIterator()
        throws ResourceNotFoundException
{
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(split.getStreamName());
    getShardIteratorRequest.setShardId(split.getShardId());

    // Explanation: when we have a sequence number from a prior read or checkpoint, always use it.
    // Otherwise, decide if starting at a timestamp or the trim horizon based on configuration.
    // If starting at a timestamp, sue the session variable ITER_START_TIMESTAMP when given, otherwise
    // fallback on starting at ITER_OFFSET_SECONDS from timestamp.
    if (lastReadSeqNo == null) {
        // Important: shard iterator type AT_TIMESTAMP requires 1.11.x or above of the AWS SDK.
        if (SessionVariables.getIterFromTimestamp(session)) {
            getShardIteratorRequest.setShardIteratorType("AT_TIMESTAMP");
            long iterStartTs = SessionVariables.getIterStartTimestamp(session);
            if (iterStartTs == 0) {
                long startTs = System.currentTimeMillis() - (SessionVariables.getIterOffsetSeconds(session) * 1000);
                getShardIteratorRequest.setTimestamp(new Date(startTs));
            }
            else {
                getShardIteratorRequest.setTimestamp(new Date(iterStartTs));
            }
        }
        else {
            getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON");
        }
    }
    else {
        getShardIteratorRequest.setShardIteratorType("AFTER_SEQUENCE_NUMBER");
        getShardIteratorRequest.setStartingSequenceNumber(lastReadSeqNo);
    }

    GetShardIteratorResult getShardIteratorResult = clientManager.getClient().getShardIterator(getShardIteratorRequest);
    shardIterator = getShardIteratorResult.getShardIterator();
}
Example #16
Source File: MockKinesisClient.java    From presto-kinesis with Apache License 2.0 5 votes vote down vote up 
@Override
public GetShardIteratorResult getShardIterator(GetShardIteratorRequest getShardIteratorRequest) throws AmazonServiceException, AmazonClientException
{
    ShardIterator iter = ShardIterator.fromStreamAndShard(getShardIteratorRequest.getStreamName(), getShardIteratorRequest.getShardId());
    if (iter != null) {
        InternalStream theStream = this.getStream(iter.streamId);
        if (theStream != null) {
            String seqAsString = getShardIteratorRequest.getStartingSequenceNumber();
            if (seqAsString != null && !seqAsString.isEmpty() && getShardIteratorRequest.getShardIteratorType().equals("AFTER_SEQUENCE_NUMBER")) {
                int sequence = Integer.parseInt(seqAsString);
                iter.recordIndex = sequence + 1;
            }
            else {
                iter.recordIndex = 100;
            }

            GetShardIteratorResult result = new GetShardIteratorResult();
            return result.withShardIterator(iter.makeString());
        }
        else {
            throw new AmazonClientException("Unknown stream or bad shard iterator!");
        }
    }
    else {
        throw new AmazonClientException("Bad stream or shard iterator!");
    }
}
Example #17
Source File: KinesisProxy.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * {@inheritDoc}
 */
@Override
public String getShardIterator(StreamShardHandle shard, String shardIteratorType, @Nullable Object startingMarker) throws InterruptedException {
	GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
		.withStreamName(shard.getStreamName())
		.withShardId(shard.getShard().getShardId())
		.withShardIteratorType(shardIteratorType);

	switch (ShardIteratorType.fromValue(shardIteratorType)) {
		case TRIM_HORIZON:
		case LATEST:
			break;
		case AT_TIMESTAMP:
			if (startingMarker instanceof Date) {
				getShardIteratorRequest.setTimestamp((Date) startingMarker);
			} else {
				throw new IllegalArgumentException("Invalid object given for GetShardIteratorRequest() when ShardIteratorType is AT_TIMESTAMP. Must be a Date object.");
			}
			break;
		case AT_SEQUENCE_NUMBER:
		case AFTER_SEQUENCE_NUMBER:
			if (startingMarker instanceof String) {
				getShardIteratorRequest.setStartingSequenceNumber((String) startingMarker);
			} else {
				throw new IllegalArgumentException("Invalid object given for GetShardIteratorRequest() when ShardIteratorType is AT_SEQUENCE_NUMBER or AFTER_SEQUENCE_NUMBER. Must be a String.");
			}
	}
	return getShardIterator(getShardIteratorRequest);
}