com.amazonaws.services.kinesis.connectors.UnmodifiableBuffer Java Examples

@SuppressWarnings("deprecation")
@Test
public void insertTest() throws Exception {
    // Set up the buffer and do sanity checks
    buffer.clear();
    buffer.consumeRecord(ITEM1_INSERT, ITEM1_INSERT.getDynamodb().getSizeBytes().intValue(), ITEM1_INSERT.getDynamodb().getSequenceNumber());
    assertEquals(ITEM1_INSERT.getDynamodb().getSequenceNumber(), buffer.getFirstSequenceNumber());
    assertEquals(ITEM1_INSERT.getDynamodb().getSequenceNumber(), buffer.getLastSequenceNumber());
    List<Record> buffered = buffer.getRecords();
    assertEquals(1, buffered.size());
    assertTrue(buffered.contains(ITEM1_INSERT));

    // Emit record
    resetAll(DYNAMODB);
    expectNew(AmazonDynamoDBAsyncClient.class, new Class<?>[] {AWSCredentialsProvider.class, ClientConfiguration.class, ExecutorService.class}, anyObject(AWSCredentialsProvider.class), anyObject(ClientConfiguration.class), anyObject(ExecutorService.class)).andReturn(DYNAMODB);
    DYNAMODB.putItemAsync(EasyMock.anyObject(PutItemRequest.class), anyObject(AsyncHandler.class));
    expectLastCall().andAnswer(SUCCESS_ANSWER);

    DYNAMODB.setEndpoint(EasyMock.anyString());
    EasyMock.expectLastCall().anyTimes();
    replayAll(DYNAMODB);
    IEmitter<Record> instance = createEmitterInstance();
    assertTrue(instance.emit(new UnmodifiableBuffer<Record>(buffer)).isEmpty());
    verifyAll();
}
 

@SuppressWarnings("deprecation")
@Test
public void modifyTest() throws Exception {
    // Set up the buffer and do sanity checks
    buffer.clear();
    buffer.consumeRecord(ITEM1_MODIFY, ITEM1_MODIFY.getDynamodb().getSizeBytes().intValue(), ITEM1_MODIFY.getDynamodb().getSequenceNumber());
    assertEquals(ITEM1_MODIFY.getDynamodb().getSequenceNumber(), buffer.getFirstSequenceNumber());
    assertEquals(ITEM1_MODIFY.getDynamodb().getSequenceNumber(), buffer.getLastSequenceNumber());
    List<Record> buffered = buffer.getRecords();
    assertEquals(1, buffered.size());
    assertTrue(buffered.contains(ITEM1_MODIFY));

    // Emit record
    resetAll(DYNAMODB);
    DYNAMODB.putItemAsync(EasyMock.anyObject(PutItemRequest.class), anyObject(AsyncHandler.class));
    expectLastCall().andAnswer(SUCCESS_ANSWER);
    expectNew(AmazonDynamoDBAsyncClient.class, new Class<?>[] {AWSCredentialsProvider.class, ClientConfiguration.class, ExecutorService.class}, anyObject(AWSCredentialsProvider.class), anyObject(ClientConfiguration.class), anyObject(ExecutorService.class)).andReturn(DYNAMODB);
    DYNAMODB.setEndpoint(EasyMock.anyString());
    EasyMock.expectLastCall().anyTimes();
    replayAll(DYNAMODB);
    IEmitter<Record> instance = createEmitterInstance();
    assertTrue(instance.emit(new UnmodifiableBuffer<Record>(buffer)).isEmpty());
    verifyAll();
}
 

@SuppressWarnings("deprecation")
@Test
public void removeTest() throws Exception {
    // Set up the buffer and do sanity checks
    buffer.clear();
    buffer.consumeRecord(ITEM1_REMOVE, ITEM1_REMOVE.getDynamodb().getSizeBytes().intValue(), ITEM1_REMOVE.getDynamodb().getSequenceNumber());
    assertEquals(ITEM1_REMOVE.getDynamodb().getSequenceNumber(), buffer.getFirstSequenceNumber());
    assertEquals(ITEM1_REMOVE.getDynamodb().getSequenceNumber(), buffer.getLastSequenceNumber());
    List<Record> buffered = buffer.getRecords();
    assertEquals(1, buffered.size());
    assertTrue(buffered.contains(ITEM1_REMOVE));

    // Emit record
    resetAll(DYNAMODB);
    DYNAMODB.deleteItemAsync(anyObject(DeleteItemRequest.class), anyObject(AsyncHandler.class));
    expectLastCall().andAnswer(SUCCESS_ANSWER);
    expectNew(AmazonDynamoDBAsyncClient.class, new Class<?>[] {AWSCredentialsProvider.class, ClientConfiguration.class, ExecutorService.class}, anyObject(AWSCredentialsProvider.class), anyObject(ClientConfiguration.class), anyObject(ExecutorService.class)).andReturn(DYNAMODB);
    DYNAMODB.setEndpoint(EasyMock.anyString());
    EasyMock.expectLastCall().anyTimes();
    replayAll(DYNAMODB);
    IEmitter<Record> instance = createEmitterInstance();
    assertTrue(instance.emit(new UnmodifiableBuffer<Record>(buffer)).isEmpty());
    verifyAll();
}
 

@SuppressWarnings("deprecation")
@Test
public void multipleRecordsEmitsTest() throws Exception {
    // Set up the buffer and do sanity checks
    buffer.clear();
    buffer.consumeRecord(ITEM1_INSERT, ITEM1_INSERT.getDynamodb().getSizeBytes().intValue(), ITEM1_INSERT.getDynamodb().getSequenceNumber());
    buffer.consumeRecord(ITEM2_INSERT, ITEM2_INSERT.getDynamodb().getSizeBytes().intValue(), ITEM2_INSERT.getDynamodb().getSequenceNumber());
    assertEquals(ITEM1_INSERT.getDynamodb().getSequenceNumber(), buffer.getFirstSequenceNumber());
    assertEquals(ITEM2_INSERT.getDynamodb().getSequenceNumber(), buffer.getLastSequenceNumber());
    List<Record> buffered = buffer.getRecords();
    assertEquals(2, buffered.size());
    assertTrue(buffered.contains(ITEM1_INSERT));
    assertTrue(buffered.contains(ITEM2_INSERT));

    // Emit record
    resetAll(DYNAMODB);
    DYNAMODB.putItemAsync(EasyMock.anyObject(PutItemRequest.class), anyObject(AsyncHandler.class));
    expectLastCall().andAnswer(SUCCESS_ANSWER).times(2);
    expectNew(AmazonDynamoDBAsyncClient.class, new Class<?>[] {AWSCredentialsProvider.class, ClientConfiguration.class, ExecutorService.class}, anyObject(AWSCredentialsProvider.class), anyObject(ClientConfiguration.class), anyObject(ExecutorService.class)).andReturn(DYNAMODB);
    DYNAMODB.setEndpoint(EasyMock.anyString());
    EasyMock.expectLastCall().anyTimes();
    replayAll(DYNAMODB);
    IEmitter<Record> instance = createEmitterInstance();
    assertTrue(instance.emit(new UnmodifiableBuffer<Record>(buffer)).isEmpty());
    verifyAll();
}
 

@Before
public void setUp() {
    // object under test
    Properties props = new Properties();
    AWSCredentialsProvider creds = createMock(AWSCredentialsProvider.class);
    configuration = new KinesisConnectorConfiguration(props, creds);

    emitter = new ElasticsearchEmitter(configuration);

    buffer = createMock(UnmodifiableBuffer.class);
    mockBulkBuilder = createMock(BulkRequestBuilder.class);
    mockIndexBuilder = createMock(IndexRequestBuilder.class);
    mockFuture = createMock(ListenableActionFuture.class);
    mockBulkResponse = createMock(BulkResponse.class);

    // overwrite the elasticseach client with a mock
    elasticsearchClientMock = createMock(TransportClient.class);

    setField(emitter, "elasticsearchClient", elasticsearchClientMock);
    // overwrite the default backoff time with 0 seconds to speed up tests
    setField(emitter, "BACKOFF_PERIOD", 0);
}
 

@Override
public List<byte[]> emit(final UnmodifiableBuffer<byte[]> buffer) throws IOException {
    List<byte[]> failed = super.emit(buffer);
    if (!failed.isEmpty()) {
        return buffer.getRecords();
    }
    Connection conn = null;
    try {
        conn = DriverManager.getConnection(redshiftURL, loginProperties);
        String s3File = getS3FileName(buffer.getFirstSequenceNumber(), buffer.getLastSequenceNumber());
        executeStatement(generateCopyStatement(s3File), conn);
        LOG.info("Successfully copied " + getNumberOfCopiedRecords(conn)
                + " records to Amazon Redshift from file s3://" + s3Bucket + "/" + s3File);
        return Collections.emptyList();
    } catch (Exception e) {
        LOG.error(e);
        return buffer.getRecords();
    } finally {
        closeConnection(conn);
    }
}
 

@Override
public List<Map<String, AttributeValue>> emit(final UnmodifiableBuffer<Map<String, AttributeValue>> buffer)
    throws IOException {
    // Map of WriteRequests to records for reference
    Map<WriteRequest, Map<String, AttributeValue>> requestMap =
            new HashMap<WriteRequest, Map<String, AttributeValue>>();
    List<Map<String, AttributeValue>> unproc = new ArrayList<Map<String, AttributeValue>>();
    // Build a batch request with a record list
    List<WriteRequest> rList = new ArrayList<WriteRequest>();
    List<Map<String, AttributeValue>> resultList;
    // Amazon DynamoDB only allows one operation per item in a bulk insertion (no duplicate items)
    Set<Map<String, AttributeValue>> uniqueItems = uniqueItems(buffer.getRecords());
    for (Map<String, AttributeValue> item : uniqueItems) {
        WriteRequest wr = new WriteRequest().withPutRequest(new PutRequest().withItem(item));
        // add to the map
        requestMap.put(wr, item);
        // add to the list of requests
        rList.add(wr);
        // Max of sixteen not to exceed maximum request size
        if (rList.size() == 16) {
            resultList = performBatchRequest(rList, requestMap);
            unproc.addAll(resultList);
            rList.clear();
        }
    }
    resultList = performBatchRequest(rList, requestMap);
    unproc.addAll(resultList);
    LOG.info("Successfully emitted " + (buffer.getRecords().size() - unproc.size()) + " records into DynamoDB.");
    return unproc;
}
 

/**
 * expect buffer flush, emit and checkpoint to happen on ShutdownReason.TERMINATE
 */
@Test
public void testShutdownTerminate() throws IOException, KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
    // reset the control to mock new behavior
    control.reset();

    // expect flush cycle.
    // Get records from buffer, emit, clear, then checkpoint
    EasyMock.expect(buffer.getRecords()).andReturn(Collections.emptyList());
    EasyMock.expect(emitter.emit(EasyMock.anyObject(UnmodifiableBuffer.class))).andReturn(
            Collections.emptyList());
    buffer.getLastSequenceNumber();
    EasyMock.expectLastCall().andReturn(null);
    buffer.clear();
    EasyMock.expectLastCall();
    checkpointer.checkpoint();
    EasyMock.expectLastCall();

    // expect shutdown to be called on emitter
    emitter.shutdown();
    EasyMock.expectLastCall();

    // Prepare controller for method call
    control.replay();

    KinesisConnectorRecordProcessor<Object, Object> kcrp = new KinesisConnectorRecordProcessor<Object, Object>(
            buffer, filter, emitter, transformer, configuration);
    kcrp.shutdown(checkpointer, ShutdownReason.TERMINATE);

    control.verify();
}
 

@Override
public List<byte[]> emit(final UnmodifiableBuffer<byte[]> buffer) throws IOException {
    // Store the contents of buffer.getRecords because superclass will
    // clear the buffer on success
    List<byte[]> failed = super.emit(buffer);
    // calls S3Emitter to write objects to Amazon S3
    if (!failed.isEmpty()) {
        return buffer.getRecords();
    }
    String s3File = getS3FileName(buffer.getFirstSequenceNumber(), buffer.getLastSequenceNumber());
    // wrap the name of the Amazon S3 file as the record data
    ByteBuffer data = ByteBuffer.wrap(s3File.getBytes());
    // Put the list of file names to the manifest Amazon Kinesis stream
    PutRecordRequest putRecordRequest = new PutRecordRequest();
    putRecordRequest.setData(data);
    putRecordRequest.setStreamName(manifestStream);
    // Use constant partition key to ensure file order
    putRecordRequest.setPartitionKey(manifestStream);
    try {
        kinesisClient.putRecord(putRecordRequest);
        LOG.info("S3ManifestEmitter emitted record downstream: " + s3File);
        return Collections.emptyList();
    } catch (Exception e) {
        LOG.error(e);
        return buffer.getRecords();
    }
}
 

@Test
@SuppressWarnings("unchecked")
public void testUnmodifiableBuffer() {
    long bytesToBuffer = 10;
    long recordsToBuffer = 10;
    String firstSequenceNumber = "firstSequenceNumber";
    String lastSequenceNumber = "lastSequenceNumber";
    List<Integer> records = new ArrayList<Integer>();

    buffer = (IBuffer<Integer>) control.createMock(IBuffer.class);
    control.reset();

    // Modifiable Buffer Behavior
    EasyMock.expect(buffer.getBytesToBuffer()).andReturn(bytesToBuffer);
    EasyMock.expect(buffer.getNumRecordsToBuffer()).andReturn(recordsToBuffer);
    EasyMock.expect(buffer.getFirstSequenceNumber()).andReturn(firstSequenceNumber);
    EasyMock.expect(buffer.getLastSequenceNumber()).andReturn(lastSequenceNumber);
    EasyMock.expect(buffer.getRecords()).andReturn(records);
    control.replay();

    UnmodifiableBuffer<Integer> umb = new UnmodifiableBuffer<Integer>(buffer);

    // Test getters behave like buffer
    assertEquals(umb.getBytesToBuffer(), bytesToBuffer);
    assertEquals(umb.getNumRecordsToBuffer(), recordsToBuffer);
    assertEquals(umb.getFirstSequenceNumber(), firstSequenceNumber);
    assertEquals(umb.getLastSequenceNumber(), lastSequenceNumber);
    List<Integer> testRecords = umb.getRecords();
    // Verify buffer methods were called
    control.verify();

    // Expect exceptions on the following operations
    exception.expect(UnsupportedOperationException.class);
    umb.consumeRecord(1, 1, "");

    exception.expect(UnsupportedOperationException.class);
    umb.clear();

    exception.expect(UnsupportedOperationException.class);
    testRecords.clear();
}
 

/**
 * {@inheritDoc}
 */
@SuppressWarnings("unchecked")
@Override
public List<Record> emit(final UnmodifiableBuffer<Record> buffer) {
    if (isShutdown) {
        if (buffer.getRecords().isEmpty()) {
            // This is OK, but not expected
            log.warn("Record processor called emit after calling shutdown. Continuing becuase buffer is empty.");
            return Collections.emptyList();
        } else {
            throw new IllegalStateException("Cannot emit records after emitter has been shutdown.");
        }
    }
    // Asynchronously process all writes, but block on the results.
    List<Record> records = buffer.getRecords();
    // Stores records that failed with a non-retryable exception
    final List<Record> failedRecords = Collections.synchronizedList(new ArrayList<Record>());
    // Queue of records to submit
    final BlockingQueue<Record> toSubmit = new LinkedBlockingQueue<Record>(records);
    // Used to detect when all requests have either succeeded or resulted in a non-retryable exception
    final CountDownLatch doneSignal = new CountDownLatch(records.size());
    final AtomicInteger retryCount = new AtomicInteger();
    boolean interrupted = false;
    try {
        while (doneSignal.getCount() > 0) {
            Record recordToSubmit = null;
            try {
                recordToSubmit = toSubmit.poll(WAIT_TIME_MS, TimeUnit.MILLISECONDS);
            } catch (InterruptedException e) {
                interrupted = true;
            }
            final Record record = recordToSubmit;
            if (null == record) {
                continue; // Check if all records have completed and if not try to poll again
            }
            // Generate the request based on the record
            AmazonWebServiceRequest request = createRequest(record);
            if (request == null) { // Should only happen if DynamoDB Streams API updates to support different operations
                                   // than {INSERT, MODIFY, REMOVE}.
                continue;
            }
            // Submit the write request based on its type
            if (request instanceof PutItemRequest) { // PUT
                getDynamodb().putItemAsync((PutItemRequest) request,
                    (AsyncHandler<PutItemRequest, PutItemResult>) getHandler(toSubmit, failedRecords, retryCount, doneSignal, record));
            } else if (request instanceof DeleteItemRequest) { // DELETE
                getDynamodb().deleteItemAsync((DeleteItemRequest) request,
                    (AsyncHandler<DeleteItemRequest, DeleteItemResult>) getHandler(toSubmit, failedRecords, retryCount, doneSignal, record));
            } else if (request instanceof UpdateItemRequest) { // UPDATE
                getDynamodb().updateItemAsync((UpdateItemRequest) request,
                    (AsyncHandler<UpdateItemRequest, UpdateItemResult>) getHandler(toSubmit, failedRecords, retryCount, doneSignal, record));
            } else { // Should only happen if DynamoDB allows a new operation other than {PutItem, DeleteItem,
                     // UpdateItem} for single item writes.
                log.warn("Unsupported DynamoDB request: " + request);
            }
        }
    } finally {
        if (interrupted) {
            Thread.currentThread().interrupt();
        }
    }
    emitCloudWatchMetrics(records, failedRecords, retryCount);
    if (!records.isEmpty()) {
        log.debug("Successfully emitted " + (records.size() - failedRecords.size()) + " records ending with sequence number "
            + buffer.getLastSequenceNumber());
    } else {
        log.debug("No records to emit");
    }
    return failedRecords;
}
 

/**
 * Test fail called when all retries done.
 */
@Test
public void testFailAfterRetryLimitReached() throws IOException, KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
    // Test Variables
    Object dummyRecord1 = new Object();
    Object dummyRecord2= new Object();
    List<Object> objectsAsList = new ArrayList<Object>();
    objectsAsList.add(dummyRecord1);
    objectsAsList.add(dummyRecord2);

    List<Object> singleObjectAsList = new ArrayList<Object>();
    singleObjectAsList.add(dummyRecord1);
    String shardId = "shardId";

    Properties props = new Properties();
    // set backoff interval to 0 to speed up test
    props.setProperty(KinesisConnectorConfiguration.PROP_BACKOFF_INTERVAL, String.valueOf(0));
    // set retry limit to allow for retry below (1)
    props.setProperty(KinesisConnectorConfiguration.PROP_RETRY_LIMIT, String.valueOf(1));
    configuration = new KinesisConnectorConfiguration(props, new DefaultAWSCredentialsProviderChain());

    // reset the control to mock new behavior
    control.reset();

    // set expectations for each record
    EasyMock.expect(transformer.toClass(EasyMock.anyObject(Record.class))).andReturn(dummyRecord1);
    EasyMock.expect(filter.keepRecord(dummyRecord1)).andReturn(true);
    buffer.consumeRecord(dummyRecord1, DEFAULT_RECORD_BYTE_SIZE, DEFAULT_SEQUENCE_NUMBER);

    EasyMock.expect(transformer.toClass(EasyMock.anyObject(Record.class))).andReturn(dummyRecord2);
    EasyMock.expect(filter.keepRecord(dummyRecord2)).andReturn(true);
    buffer.consumeRecord(dummyRecord2, DEFAULT_RECORD_BYTE_SIZE, DEFAULT_SEQUENCE_NUMBER);

    EasyMock.expect(buffer.shouldFlush()).andReturn(true);

    // call Buffer.getRecords
    EasyMock.expect(buffer.getRecords()).andReturn(objectsAsList);

    // Transform back
    EasyMock.expect(transformer.fromClass(dummyRecord1)).andReturn(dummyRecord1);
    EasyMock.expect(transformer.fromClass(dummyRecord2)).andReturn(dummyRecord2);

    // Emitter behavior:
    // one call to emit which fails (test a transient issue), and then returns empty on second call

    // uses the original list (i.e. emitItems)
    UnmodifiableBuffer<Object> unmodBuffer = new UnmodifiableBuffer<>(buffer, objectsAsList);
    EasyMock.expect(emitter.emit(EasyMock.eq(unmodBuffer))).andReturn(singleObjectAsList);

    // only one retry, so now we should expect fail to be called.
    emitter.fail(singleObjectAsList);

    // Done, so expect buffer clear and checkpoint
    buffer.getLastSequenceNumber();
    EasyMock.expectLastCall().andReturn(DEFAULT_SEQUENCE_NUMBER);
    buffer.clear();
    EasyMock.expectLastCall();
    checkpointer.checkpoint(DEFAULT_SEQUENCE_NUMBER);
    EasyMock.expectLastCall();

    // Initialize class under test
    KinesisConnectorRecordProcessor<Object, Object> kcrp = new KinesisConnectorRecordProcessor<Object, Object>(
            buffer, filter, emitter, transformer, configuration);
    kcrp.initialize(shardId);

    // Prepare controller for method call
    control.replay();

    // call method
    kcrp.processRecords(getDummyRecordList(objectsAsList.size()), checkpointer);

    control.verify();
}
 

/**
 * Test retry logic only retries unprocessed/failed records
 */
@Test
public void testRetryBehavior() throws IOException, KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
    // Test Variables
    Object dummyRecord1 = new Object();
    Object dummyRecord2= new Object();
    List<Object> objectsAsList = new ArrayList<Object>();
    objectsAsList.add(dummyRecord1);
    objectsAsList.add(dummyRecord2);

    List<Object> singleObjectAsList = new ArrayList<Object>();
    singleObjectAsList.add(dummyRecord1);
    String shardId = "shardId";

    Properties props = new Properties();
    // set backoff interval to 0 to speed up test
    props.setProperty(KinesisConnectorConfiguration.PROP_BACKOFF_INTERVAL, String.valueOf(0));
    // set retry limit to allow for retry below
    props.setProperty(KinesisConnectorConfiguration.PROP_RETRY_LIMIT, String.valueOf(2));
    configuration = new KinesisConnectorConfiguration(props, new DefaultAWSCredentialsProviderChain());

    // reset the control to mock new behavior
    control.reset();

    // set expectations for each record
    EasyMock.expect(transformer.toClass(EasyMock.anyObject(Record.class))).andReturn(dummyRecord1);
    EasyMock.expect(filter.keepRecord(dummyRecord1)).andReturn(true);
    buffer.consumeRecord(dummyRecord1, DEFAULT_RECORD_BYTE_SIZE, DEFAULT_SEQUENCE_NUMBER);

    EasyMock.expect(transformer.toClass(EasyMock.anyObject(Record.class))).andReturn(dummyRecord2);
    EasyMock.expect(filter.keepRecord(dummyRecord2)).andReturn(true);
    buffer.consumeRecord(dummyRecord2, DEFAULT_RECORD_BYTE_SIZE, DEFAULT_SEQUENCE_NUMBER);

    EasyMock.expect(buffer.shouldFlush()).andReturn(true);

    // call Buffer.getRecords
    EasyMock.expect(buffer.getRecords()).andReturn(objectsAsList);

    // Transform back
    EasyMock.expect(transformer.fromClass(dummyRecord1)).andReturn(dummyRecord1);
    EasyMock.expect(transformer.fromClass(dummyRecord2)).andReturn(dummyRecord2);

    // Emitter behavior:
    // one call to emit which fails (test a transient issue), and then returns empty on second call

    // uses the original list (i.e. emitItems)
    UnmodifiableBuffer<Object> unmodBuffer = new UnmodifiableBuffer<>(buffer, objectsAsList);
    EasyMock.expect(emitter.emit(EasyMock.eq(unmodBuffer))).andReturn(singleObjectAsList);
    // uses the returned list (i.e. unprocessed)
    unmodBuffer = new UnmodifiableBuffer<>(buffer, singleObjectAsList);
    EasyMock.expect(emitter.emit(EasyMock.eq(unmodBuffer))).andReturn(Collections.emptyList());

    // Done, so expect buffer clear and checkpoint
    buffer.getLastSequenceNumber();
    EasyMock.expectLastCall().andReturn(DEFAULT_SEQUENCE_NUMBER);
    buffer.clear();
    EasyMock.expectLastCall();
    checkpointer.checkpoint(DEFAULT_SEQUENCE_NUMBER);
    EasyMock.expectLastCall();

    // Initialize class under test
    KinesisConnectorRecordProcessor<Object, Object> kcrp = new KinesisConnectorRecordProcessor<Object, Object>(
            buffer, filter, emitter, transformer, configuration);
    kcrp.initialize(shardId);

    // Prepare controller for method call
    control.replay();

    // call method
    kcrp.processRecords(getDummyRecordList(objectsAsList.size()), checkpointer);

    control.verify();
}
 

/**
  * Test process records under normal conditions but with batch processor.
 */
@Test
@SuppressWarnings("unchecked")
public void testProcessBatchedRecords() throws ThrottlingException, ShutdownException, IOException,
KinesisClientLibDependencyException, InvalidStateException {
    // Test Variables
    Object dummyRecord1 = new Object();
    Object dummyRecord2= new Object();
    List<Object> dummyCollection = new ArrayList<>();
    dummyCollection.add(dummyRecord1);
    dummyCollection.add(dummyRecord2);

    // 5 Kinesis records, with 2 objects per. So 10 total records
    int numRecords = 5;
    int numTotalRecords = numRecords * 2;
    String shardId = "shardId";
    // Override existing transformer with a collection transformer
    ICollectionTransformer<Object,Object> collectionTransformer = control.createMock(ICollectionTransformer.class);

    // reset the control to mock new behavior
    control.reset();

    // Transformer Behavior:
    // transform numRecords records into dummyRecord objects
    EasyMock.expect(collectionTransformer.toClass(EasyMock.anyObject(Record.class))).andReturn(dummyCollection);
    EasyMock.expectLastCall().times(numRecords -1).andReturn(dummyCollection);

    // Filter Behavior:
    // return true for all records
    EasyMock.expect(filter.keepRecord(dummyRecord1)).andReturn(true);
    EasyMock.expectLastCall().times(numRecords -1).andReturn(true);
    EasyMock.expect(filter.keepRecord(dummyRecord2)).andReturn(true);
    EasyMock.expectLastCall().times(numRecords -1).andReturn(true);

    // Mock arguments
    Object o = EasyMock.anyObject();
    int buf = EasyMock.anyInt();

    // Buffer Behavior:
    // consume numRecords dummy records
    buffer.consumeRecord(o, buf, EasyMock.anyObject(String.class));
    EasyMock.expectLastCall().times(numTotalRecords);
    buffer.getLastSequenceNumber();
    EasyMock.expectLastCall().andReturn(DEFAULT_SEQUENCE_NUMBER);
    buffer.clear();
    EasyMock.expectLastCall();

    // check full buffer and return true
    EasyMock.expect(buffer.shouldFlush()).andReturn(true);

    // call buffer.getRecords
    EasyMock.expect(buffer.getRecords()).andReturn(Collections.emptyList());

    // Emitter behavior:
    // one call to emit
    EasyMock.expect(emitter.emit(EasyMock.anyObject(UnmodifiableBuffer.class))).andReturn(
            Collections.emptyList());

    // Checkpointer Behavior:
    // one call to checkpoint
    checkpointer.checkpoint(DEFAULT_SEQUENCE_NUMBER);
    EasyMock.expectLastCall();

    // Initialize class under test
    KinesisConnectorRecordProcessor<Object, Object> kcrp = new KinesisConnectorRecordProcessor<Object, Object>(
            buffer, filter, emitter, collectionTransformer, configuration);
    kcrp.initialize(shardId);

    // Prepare controller for method call
    control.replay();

    // call method
    kcrp.processRecords(getDummyRecordList(numRecords), checkpointer);

    control.verify();
}
 

/**
 * Test emitter throws exception upon processing.
 */
@Test
@SuppressWarnings("unchecked")
public void testFailRecords() throws IOException, KinesisClientLibDependencyException,
        InvalidStateException, ThrottlingException, ShutdownException {
    Object dummyRecord = new Object();
    int numRecords = 5;
    String shardId = "shardId";

    // reset the control to mock new behavior
    control.reset();

    // Transformer Behavior:
    // transform numRecords records into dummyRecord objects
    EasyMock.expect(transformer.toClass(EasyMock.anyObject(Record.class))).andReturn(dummyRecord);
    EasyMock.expectLastCall().times(numRecords - 1).andReturn(dummyRecord);

    // Filter Behavior:
    // return true for all records
    EasyMock.expect(filter.keepRecord(dummyRecord)).andReturn(true);
    EasyMock.expectLastCall().times(numRecords -1).andReturn(true);

    // Mock arguments
    Object o = EasyMock.anyObject();
    int buf = EasyMock.anyInt();
    String seq = EasyMock.anyObject(String.class);

    // Buffer Behavior:
    // consume numRecords dummy records
    buffer.consumeRecord(o, buf, seq);
    EasyMock.expectLastCall().times(numRecords);

    // check full buffer and return true
    EasyMock.expect(buffer.shouldFlush()).andReturn(true);

    // call buffer.getRecords
    EasyMock.expect(buffer.getRecords()).andReturn(Collections.emptyList());

    // Emitter behavior:
    // one call to emit
    EasyMock.expect(emitter.emit(EasyMock.anyObject(UnmodifiableBuffer.class))).andThrow(
            new IOException());
    emitter.fail(EasyMock.anyObject(List.class));
    EasyMock.expectLastCall();

    // Initialize class under test
    KinesisConnectorRecordProcessor<Object, Object> kcrp = new KinesisConnectorRecordProcessor<Object, Object>(
            buffer, filter, emitter, transformer, configuration);
    kcrp.initialize(shardId);

    // Prepare controller for method call
    control.replay();

    // call method
    kcrp.processRecords(getDummyRecordList(numRecords), checkpointer);

    control.verify();
}
 

/**
  * Test process records under normal conditions.
  */
@Test
@SuppressWarnings("unchecked")
public void testProcessRecords() throws ThrottlingException, ShutdownException, IOException,
        KinesisClientLibDependencyException, InvalidStateException {
    // Test Variables
    Object dummyRecord = new Object();
    int numRecords = 5;
    String shardId = "shardId";

    // reset the control to mock new behavior
    control.reset();

    // Transformer Behavior:
    // transform numRecords records into dummyRecord objects
    EasyMock.expect(transformer.toClass(EasyMock.anyObject(Record.class))).andReturn(dummyRecord);
    EasyMock.expectLastCall().times(numRecords -1).andReturn(dummyRecord);

    // Filter Behavior:
    // return true for all records
    EasyMock.expect(filter.keepRecord(dummyRecord)).andReturn(true);
    EasyMock.expectLastCall().times(numRecords -1).andReturn(true);

    // Mock arguments
    Object o = EasyMock.anyObject();
    int buf = EasyMock.anyInt();

    // Buffer Behavior:
    // consume numRecords dummy records
    buffer.consumeRecord(o, buf, EasyMock.anyObject(String.class));
    EasyMock.expectLastCall().times(numRecords);
    buffer.getLastSequenceNumber();
    EasyMock.expectLastCall().andReturn(DEFAULT_SEQUENCE_NUMBER);
    buffer.clear();
    EasyMock.expectLastCall();

    // check full buffer and return true
    EasyMock.expect(buffer.shouldFlush()).andReturn(true);

    // call buffer.getRecords
    List<Object> objects = new ArrayList<>();
    Object item = new Object();
    objects.add(item);
    EasyMock.expect(buffer.getRecords()).andReturn(objects);
    EasyMock.expect(transformer.fromClass(item)).andReturn(item);

    // Emitter behavior:
    // one call to emit
    EasyMock.expect(emitter.emit(EasyMock.anyObject(UnmodifiableBuffer.class))).andReturn(
            Collections.emptyList());

    // Checkpointer Behavior:
    // one call to checkpoint
    checkpointer.checkpoint(DEFAULT_SEQUENCE_NUMBER);
    EasyMock.expectLastCall();

    // Initialize class under test
    KinesisConnectorRecordProcessor<Object, Object> kcrp = new KinesisConnectorRecordProcessor<Object, Object>(
            buffer, filter, emitter, transformer, configuration);
    kcrp.initialize(shardId);

    // Prepare controller for method call
    control.replay();

    // call method
    kcrp.processRecords(getDummyRecordList(numRecords), checkpointer);

    control.verify();
}
 

@Override
public List<String> emit(final UnmodifiableBuffer<String> buffer) throws IOException {
    List<String> records = buffer.getRecords();
    Connection conn = null;

    String manifestFileName = getManifestFile(records);
    // Copy to Amazon Redshift using manifest file
    try {
        conn = DriverManager.getConnection(redshiftURL, loginProps);
        conn.setAutoCommit(false);
        List<String> deduplicatedRecords = checkForExistingFiles(conn, records);
        if (deduplicatedRecords.isEmpty()) {
            LOG.info("All the files in this set were already copied to Redshift.");
            // All of these files were already written
            rollbackConnection(conn);
            return Collections.emptyList();
        }

        if (deduplicatedRecords.size() != records.size()) {
            manifestFileName = getManifestFile(deduplicatedRecords);
        }
        LOG.info("Writing manifest file " + manifestFileName + " to Amazon S3.");
        writeManifestToS3(manifestFileName, deduplicatedRecords);
        LOG.info("Inserting " + deduplicatedRecords.size() + " rows into the files table.");
        insertRecords(conn, deduplicatedRecords);
        LOG.info("Initiating Amazon Redshift manifest copy of " + deduplicatedRecords.size() + " files.");
        redshiftCopy(conn, manifestFileName);
        conn.commit();
        LOG.info("Successful Amazon Redshift manifest copy of " + getNumberOfCopiedRecords(conn) + " records from "
                + deduplicatedRecords.size() + " files using manifest s3://" + s3Bucket + "/"
                + getManifestFile(records));
        return Collections.emptyList();
    } catch (Exception e) {
        LOG.error("Error emitting data to Amazon Redshift for manifest file name "
                + manifestFileName + ". Failing this emit attempt.", e);
        rollbackConnection(conn);
        return buffer.getRecords();
    } finally {
        closeConnection(conn);
    }
}
 

/**
 * Emits records to elasticsearch.
 * 1. Adds each record to a bulk index request, conditionally adding version, ttl or create if they were set in the
 * transformer.
 * 2. Executes the bulk request, returning any record specific failures to be retried by the connector library
 * pipeline, unless
 * outlined below.
 * 
 * Record specific failures (noted in the failure.getMessage() string)
 * - DocumentAlreadyExistsException means the record has create set to true, but a document already existed at the
 * specific index/type/id.
 * - VersionConflictEngineException means the record has a specific version number that did not match what existed
 * in elasticsearch.
 * To guarantee in order processing by the connector, when putting data use the same partition key for objects going
 * to the same
 * index/type/id and set sequence number for ordering.
 * - In either case, the emitter will assume that the record would fail again in the future and thus will not return
 * the record to
 * be retried.
 * 
 * Bulk request failures
 * - NoNodeAvailableException means the TransportClient could not connect to the cluster.
 * - A general Exception catches any other unexpected behavior.
 * - In either case the emitter will continue making attempts until the issue has been resolved. This is to ensure
 * that no data
 * loss occurs and simplifies restarting the application once issues have been fixed.
 */
@Override
public List<ElasticsearchObject> emit(UnmodifiableBuffer<ElasticsearchObject> buffer) throws IOException {
    List<ElasticsearchObject> records = buffer.getRecords();
    if (records.isEmpty()) {
        return Collections.emptyList();
    }

    BulkRequestBuilder bulkRequest = elasticsearchClient.prepareBulk();
    for (ElasticsearchObject record : records) {
        IndexRequestBuilder indexRequestBuilder =
                elasticsearchClient.prepareIndex(record.getIndex(), record.getType(), record.getId());
        indexRequestBuilder.setSource(record.getSource());
        Long version = record.getVersion();
        if (version != null) {
            indexRequestBuilder.setVersion(version);
        }
        Long ttl = record.getTtl();
        if (ttl != null) {
            indexRequestBuilder.setTTL(ttl);
        }
        Boolean create = record.getCreate();
        if (create != null) {
            indexRequestBuilder.setCreate(create);
        }
        bulkRequest.add(indexRequestBuilder);
    }

    while (true) {
        try {
            BulkResponse bulkResponse = bulkRequest.execute().actionGet();

            BulkItemResponse[] responses = bulkResponse.getItems();
            List<ElasticsearchObject> failures = new ArrayList<ElasticsearchObject>();
            int numberOfSkippedRecords = 0;
            for (int i = 0; i < responses.length; i++) {
                if (responses[i].isFailed()) {
                    LOG.error("Record failed with message: " + responses[i].getFailureMessage());
                    Failure failure = responses[i].getFailure();
                    if (failure.getMessage().contains("DocumentAlreadyExistsException")
                            || failure.getMessage().contains("VersionConflictEngineException")) {
                        numberOfSkippedRecords++;
                    } else {
                        failures.add(records.get(i));
                    }
                }
            }
            LOG.info("Emitted " + (records.size() - failures.size() - numberOfSkippedRecords)
                    + " records to Elasticsearch");
            if (!failures.isEmpty()) {
                printClusterStatus();
                LOG.warn("Returning " + failures.size() + " records as failed");
            }
            return failures;
        } catch (NoNodeAvailableException nnae) {
            LOG.error("No nodes found at " + elasticsearchEndpoint + ":" + elasticsearchPort + ". Retrying in "
                    + BACKOFF_PERIOD + " milliseconds", nnae);
            sleep(BACKOFF_PERIOD);
        } catch (Exception e) {
            LOG.error("ElasticsearchEmitter threw an unexpected exception ", e);
            sleep(BACKOFF_PERIOD);
        }
    }

}
 

@Override
public List<Map<String,String>> emit(final UnmodifiableBuffer<Map<String,String>> buffer) throws IOException {
	List<Map<String, String>> records = buffer.getRecords();
	ListIterator<Map<String, String>> iterator = records.listIterator();
	List<Put> batch = new ArrayList<Put>();    	
	HashMap<String, String> hashMap = (HashMap<String, String>) iterator.next();   	
	while (iterator.hasNext()) {
		//start with the row key followed by column family
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("user"), Bytes.toBytes("userid"),Bytes.toBytes(hashMap.get("userid"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("user"), Bytes.toBytes("firstname"),Bytes.toBytes(hashMap.get("firstname"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("user"), Bytes.toBytes("lastname"),Bytes.toBytes(hashMap.get("lastname"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("address"), Bytes.toBytes("city"),Bytes.toBytes(hashMap.get("city"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("address"), Bytes.toBytes("state"),Bytes.toBytes(hashMap.get("state"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("contact"), Bytes.toBytes("email"),Bytes.toBytes(hashMap.get("email"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("contact"), Bytes.toBytes("phone"),Bytes.toBytes(hashMap.get("phone"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("likes"), Bytes.toBytes("likesports"),Bytes.toBytes(hashMap.get("likesports"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("likes"), Bytes.toBytes("liketheatre"),Bytes.toBytes(hashMap.get("liketheatre"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("likes"), Bytes.toBytes("likeconcerts"),Bytes.toBytes(hashMap.get("likeconcerts"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("likes"), Bytes.toBytes("likejazz"),Bytes.toBytes(hashMap.get("likejazz"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("likes"), Bytes.toBytes("likeclassical"),Bytes.toBytes(hashMap.get("likeclassical"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("likes"), Bytes.toBytes("likeopera"),Bytes.toBytes(hashMap.get("likeopera"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("likes"), Bytes.toBytes("likerock"),Bytes.toBytes(hashMap.get("likerock"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("likes"), Bytes.toBytes("likevegas"),Bytes.toBytes(hashMap.get("likevegas"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("likes"), Bytes.toBytes("likebroadway"),Bytes.toBytes(hashMap.get("likebroadway"))));
		batch.add(new Put(Bytes.toBytes(hashMap.get("username")))   
		.add(Bytes.toBytes("likes"), Bytes.toBytes("likemusicals"),Bytes.toBytes(hashMap.get("likemusicals"))));
		
		hashMap = (HashMap<String, String>) iterator.next();	
	}
	LOG.info("EMIT: " + "records ....."+batch.size());
	HBaseUtils.addRecords(hbaseTableName, emrPublicDns, hbaseRestPort, batch);
	return Collections.emptyList();
	//return records;
}
 

/**
 * Invoked when the buffer is full. This method emits the set of filtered records. It should
 * return a list of records that were not emitted successfully. Returning
 * Collections.emptyList() is considered a success.
 * 
 * @param buffer
 *        The full buffer of records
 * @throws IOException
 *         A failure was reached that is not recoverable, no retry will occur and the fail
 *         method will be called
 * @return A list of records that failed to emit to be retried
 */
List<T> emit(UnmodifiableBuffer<T> buffer) throws IOException;