Java Code Examples for com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput#getRecords()

The following examples show how to use com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput#getRecords() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StreamsRecordProcessor.java    From pulsar with Apache License 2.0 6 votes vote down vote up 
@Override
public void processRecords(ProcessRecordsInput processRecordsInput) {

    log.debug("Processing {} records from {}", processRecordsInput.getRecords().size(), kinesisShardId);

    for (Record record : processRecordsInput.getRecords()) {
        try {
            queue.put(new StreamsRecord(record));
        } catch (InterruptedException e) {
            log.warn("unable to create KinesisRecord ", e);
        }
    }

    // Checkpoint once every checkpoint interval.
    if (System.nanoTime() > nextCheckpointTimeInNanos) {
        checkpoint(processRecordsInput.getCheckpointer());
        nextCheckpointTimeInNanos = System.nanoTime() + checkpointInterval;
    }
}
Example 2
Source File: StreamsRecordProcessor.java    From dynamo-cassandra-proxy with Apache License 2.0 5 votes vote down vote up 
@Override
public void processRecords(ProcessRecordsInput processRecordsInput) {
    for (Record record : processRecordsInput.getRecords()) {
        String data = new String(record.getData().array(), Charset.forName("UTF-8"));
        System.out.println(data);
        if (record instanceof RecordAdapter) {
            com.amazonaws.services.dynamodbv2.model.Record streamRecord = ((RecordAdapter) record)
                    .getInternalObject();

            switch (streamRecord.getEventName()) {
                case "INSERT": case "MODIFY":
                    Map<String, AttributeValue> items = streamRecord.getDynamodb().getNewImage();
                    PutItemRequest putItemRequest = new PutItemRequest().withTableName(tableName).withItem(items);
                    dynamoDBClient.putItem(putItemRequest);
                    break;
                case "REMOVE":
                    Map<String, AttributeValue> keys = streamRecord.getDynamodb().getKeys();
                    DeleteItemRequest deleteItemRequest = new DeleteItemRequest().withTableName(tableName).withKey(keys);
                    dynamoDBClient.deleteItem(deleteItemRequest);
            }
        }
        checkpointCounter += 1;
        if (checkpointCounter % 10 == 0) {
            try {
                processRecordsInput.getCheckpointer().checkpoint();
            }
            catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

}
Example 3
Source File: KinesisSpanProcessor.java    From zipkin-aws with Apache License 2.0 5 votes vote down vote up 
@Override
public void processRecords(ProcessRecordsInput processRecordsInput) {
  for (Record record : processRecordsInput.getRecords()) {
    byte[] serialized = record.getData().array();
    metrics.incrementMessages();
    metrics.incrementBytes(serialized.length);
    collector.acceptSpans(serialized, NOOP); // async
  }
}
Example 4
Source File: StreamsRecordProcessor.java    From aws-doc-sdk-examples with Apache License 2.0 5 votes vote down vote up 
@Override
public void processRecords(ProcessRecordsInput processRecordsInput) {
    for (Record record : processRecordsInput.getRecords()) {
        String data = new String(record.getData().array(), Charset.forName("UTF-8"));
        System.out.println(data);
        if (record instanceof RecordAdapter) {
            com.amazonaws.services.dynamodbv2.model.Record streamRecord = ((RecordAdapter) record)
                    .getInternalObject();

            switch (streamRecord.getEventName()) {
                case "INSERT":
                case "MODIFY":
                    StreamsAdapterDemoHelper.putItem(dynamoDBClient, tableName,
                                                     streamRecord.getDynamodb().getNewImage());
                    break;
                case "REMOVE":
                    StreamsAdapterDemoHelper.deleteItem(dynamoDBClient, tableName,
                                                        streamRecord.getDynamodb().getKeys().get("Id").getN());
            }
        }
        checkpointCounter += 1;
        if (checkpointCounter % 10 == 0) {
            try {
                processRecordsInput.getCheckpointer().checkpoint();
            }
            catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
    
}
Example 5
Source File: KinesisRecordProcessor.java    From samza with Apache License 2.0 5 votes vote down vote up 
/**
 * Process data records. The Amazon Kinesis Client Library will invoke this method to deliver data records to the
 * application. Upon fail over, the new instance will get records with sequence number greater than the checkpoint
 * position for each partition key.
 *
 * @param processRecordsInput Provides the records to be processed as well as information and capabilities related
 *        to them (eg checkpointing).
 */
@Override
public void processRecords(ProcessRecordsInput processRecordsInput) {
  // KCL does not send any records to the processor that was shutdown.
  Validate.isTrue(!shutdownRequested,
      String.format("KCL returned records after shutdown is called on the processor %s.", this));
  // KCL aways gives reference to the same checkpointer instance for a given processor instance.
  checkpointer = processRecordsInput.getCheckpointer();
  List<Record> records = processRecordsInput.getRecords();
  // Empty records are expected when KCL config has CallProcessRecordsEvenForEmptyRecordList set to true.
  if (!records.isEmpty()) {
    lastProcessedRecordSeqNumber = new ExtendedSequenceNumber(records.get(records.size() - 1).getSequenceNumber());
    listener.onReceiveRecords(ssp, records, processRecordsInput.getMillisBehindLatest());
  }
}