Java Code Examples for com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer#checkpoint()

The following examples show how to use com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer#checkpoint() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: StreamSetsRecordProcessor.java From datacollector with Apache License 2.0

6 votes

private void retryCheckpoint(IRecordProcessorCheckpointer checkpointer, Record checkpointRecord) throws InvalidStateException, ShutdownException, InterruptedException {
  if (throttleMaxRetries > 0) {
    LOG.debug("Retry checkpointing batch at record: {}", checkpointRecord.toString());
    int retryCount = 0;
    boolean success = false;

    while (retryCount < throttleMaxRetries && !success) {
      Thread.sleep(throttleWaitTime);
      try {
        checkpointer.checkpoint(checkpointRecord);
        success = true;
      } catch (ThrottlingException te) {}
      retryCount++;
    }

    if (success) {
      LOG.debug("Successfully checkpointed at record {}, after {} retries.", checkpointRecord.toString(), retryCount);
    } else {
      LOG.debug("Could not checkpoint batch at record {} after {} retries.", checkpointRecord.toString(), retryCount);
    }
  }
}

Example 2

Source File: MyRecordProcessor.java From aws-big-data-blog with Apache License 2.0

6 votes

@Override
public void processRecords(List<Record> records,
		IRecordProcessorCheckpointer checkpointer) {
	LOG.info(String.format("Received %s Records", records.size()));
	
	// add a call to your business logic here!
	//
	// myLinkedClasses.doSomething(records)
	//
	//
	try {
		checkpointer.checkpoint();
	} catch (KinesisClientLibDependencyException | InvalidStateException
			| ThrottlingException | ShutdownException e) {
		e.printStackTrace();
		super.shutdown(checkpointer, ShutdownReason.ZOMBIE);
	}
}

Example 3

Source File: KinesisConnectorRecordProcessor.java From amazon-kinesis-connectors with Apache License 2.0

6 votes

@Override
public void shutdown(IRecordProcessorCheckpointer checkpointer, ShutdownReason reason) {
    LOG.info("Shutting down record processor with shardId: " + shardId + " with reason " + reason);
    if (isShutdown) {
        LOG.warn("Record processor for shardId: " + shardId + " has been shutdown multiple times.");
        return;
    }
    switch (reason) {
        case TERMINATE:
            emit(checkpointer, transformToOutput(buffer.getRecords()));
            try {
                checkpointer.checkpoint();
            } catch (KinesisClientLibDependencyException | InvalidStateException | ThrottlingException | ShutdownException e) {
                LOG.error(e);
            }
            break;
        case ZOMBIE:
            break;
        default:
            throw new IllegalStateException("invalid shutdown reason");
    }
    emitter.shutdown();
    isShutdown = true;
}

Example 4

Source File: StreamsRecordProcessor.java From aws-dynamodb-examples with Apache License 2.0

5 votes

@Override
public void processRecords(List<Record> records,
        IRecordProcessorCheckpointer checkpointer) {
    for(Record record : records) {
        String data = new String(record.getData().array(), Charset.forName("UTF-8"));
        System.out.println(data);
        if(record instanceof RecordAdapter) {
            com.amazonaws.services.dynamodbv2.model.Record streamRecord = ((RecordAdapter) record).getInternalObject();
            
            switch(streamRecord.getEventName()) { 
            case "INSERT" : case "MODIFY" :
                StreamsAdapterDemoHelper.putItem(dynamoDBClient, tableName, streamRecord.getDynamodb().getNewImage());
                break;
            case "REMOVE" :
                StreamsAdapterDemoHelper.deleteItem(dynamoDBClient, tableName, streamRecord.getDynamodb().getKeys().get("Id").getN());
            }
        }
        checkpointCounter += 1;
        if(checkpointCounter % 10 == 0) {
            try {
                checkpointer.checkpoint();
            } catch(Exception e) {
                e.printStackTrace();
            }
        }
    }

}

Example 5

Source File: StreamsRecordProcessor.java From aws-dynamodb-examples with Apache License 2.0

5 votes

@Override
public void shutdown(IRecordProcessorCheckpointer checkpointer,
        ShutdownReason reason) {
    if(reason == ShutdownReason.TERMINATE) {
        try {
            checkpointer.checkpoint();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

Example 6

Source File: DynamoDBTableReplicator.java From podyn with Apache License 2.0

4 votes

protected IRecordProcessor createStreamProcessor() {
	return new IRecordProcessor() {

		@Override
		public void initialize(InitializationInput initializationInput) {
		}

		public List<Record> extractDynamoStreamRecords(List<com.amazonaws.services.kinesis.model.Record> kinesisRecords) {
			List<Record> dynamoRecords = new ArrayList<>(kinesisRecords.size());

			for(com.amazonaws.services.kinesis.model.Record kinesisRecord : kinesisRecords) {
				if (kinesisRecord instanceof RecordAdapter) {
					Record dynamoRecord = ((RecordAdapter) kinesisRecord).getInternalObject();
					dynamoRecords.add(dynamoRecord);
				}
			}

			return dynamoRecords;
		}

		@Override
		public void processRecords(ProcessRecordsInput processRecordsInput) {
			List<Record> records = extractDynamoStreamRecords(processRecordsInput.getRecords());

			DynamoDBTableReplicator.this.processRecords(records);

			checkpoint(processRecordsInput.getCheckpointer());
		}

		@Override
		public void shutdown(ShutdownInput shutdownInput) {
			if (shutdownInput.getShutdownReason() == ShutdownReason.TERMINATE) {
				checkpoint(shutdownInput.getCheckpointer());
			}
		}

		void checkpoint(IRecordProcessorCheckpointer checkpointer) {
			try {
				checkpointer.checkpoint();
			} catch (KinesisClientLibDependencyException|InvalidStateException|ThrottlingException|ShutdownException e) {
				LOG.warn(e);
			}
		}
	};
}