com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput Java Examples

@Test
public void collectorFailsWhenRecordEncodedAsSingleSpan() {
  Span span = TestObjects.LOTS_OF_SPANS[0];
  byte[] encodedSpan = SpanBytesEncoder.THRIFT.encode(span);
  Record kinesisRecord = new Record().withData(ByteBuffer.wrap(encodedSpan));
  ProcessRecordsInput kinesisInput =
      new ProcessRecordsInput().withRecords(Collections.singletonList(kinesisRecord));

  kinesisSpanProcessor.processRecords(kinesisInput);

  assertThat(storage.spanStore().getTraces().size()).isEqualTo(0);

  assertThat(metrics.messages()).isEqualTo(1);
  assertThat(metrics.messagesDropped()).isEqualTo(1);
  assertThat(metrics.bytes()).isEqualTo(encodedSpan.length);
}
 

@Override
public void processRecords(ProcessRecordsInput processRecordsInput) {

    log.debug("Processing {} records from {}", processRecordsInput.getRecords().size(), kinesisShardId);

    for (Record record : processRecordsInput.getRecords()) {
        try {
            queue.put(new StreamsRecord(record));
        } catch (InterruptedException e) {
            log.warn("unable to create KinesisRecord ", e);
        }
    }

    // Checkpoint once every checkpoint interval.
    if (System.nanoTime() > nextCheckpointTimeInNanos) {
        checkpoint(processRecordsInput.getCheckpointer());
        nextCheckpointTimeInNanos = System.nanoTime() + checkpointInterval;
    }
}
 

static Map<KinesisRecordProcessor, List<Record>> generateRecords(int numRecordsPerShard,
    List<KinesisRecordProcessor> processors) {
  Map<KinesisRecordProcessor, List<Record>> processorRecordMap = new HashMap<>();
  processors.forEach(processor -> {
    try {
      // Create records and call process records
      IRecordProcessorCheckpointer checkpointer = Mockito.mock(IRecordProcessorCheckpointer.class);
      doNothing().when(checkpointer).checkpoint(anyString());
      doNothing().when(checkpointer).checkpoint();
      ProcessRecordsInput processRecordsInput = Mockito.mock(ProcessRecordsInput.class);
      when(processRecordsInput.getCheckpointer()).thenReturn(checkpointer);
      when(processRecordsInput.getMillisBehindLatest()).thenReturn(1000L);
      List<Record> inputRecords = createRecords(numRecordsPerShard);
      processorRecordMap.put(processor, inputRecords);
      when(processRecordsInput.getRecords()).thenReturn(inputRecords);
      processor.processRecords(processRecordsInput);
    } catch (ShutdownException | InvalidStateException ex) {
      throw new RuntimeException(ex);
    }
  });
  return processorRecordMap;
}
 

@Override
public void processRecords(ProcessRecordsInput processRecordsInput) {
    for (Record record : processRecordsInput.getRecords()) {
        String data = new String(record.getData().array(), Charset.forName("UTF-8"));
        System.out.println(data);
        if (record instanceof RecordAdapter) {
            com.amazonaws.services.dynamodbv2.model.Record streamRecord = ((RecordAdapter) record)
                    .getInternalObject();

            switch (streamRecord.getEventName()) {
                case "INSERT": case "MODIFY":
                    Map<String, AttributeValue> items = streamRecord.getDynamodb().getNewImage();
                    PutItemRequest putItemRequest = new PutItemRequest().withTableName(tableName).withItem(items);
                    dynamoDBClient.putItem(putItemRequest);
                    break;
                case "REMOVE":
                    Map<String, AttributeValue> keys = streamRecord.getDynamodb().getKeys();
                    DeleteItemRequest deleteItemRequest = new DeleteItemRequest().withTableName(tableName).withKey(keys);
                    dynamoDBClient.deleteItem(deleteItemRequest);
            }
        }
        checkpointCounter += 1;
        if (checkpointCounter % 10 == 0) {
            try {
                processRecordsInput.getCheckpointer().checkpoint();
            }
            catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

}
 

@Override
public void processRecords(ProcessRecordsInput processRecordsInput) {
  for (Record record : processRecordsInput.getRecords()) {
    byte[] serialized = record.getData().array();
    metrics.incrementMessages();
    metrics.incrementBytes(serialized.length);
    collector.acceptSpans(serialized, NOOP); // async
  }
}
 

void messageWithMultipleSpans(SpanBytesEncoder encoder) {
  byte[] message = encoder.encodeList(spans);

  List<Record> records = Arrays.asList(new Record().withData(ByteBuffer.wrap(message)));
  kinesisSpanProcessor.processRecords(new ProcessRecordsInput().withRecords(records));

  assertThat(storage.spanStore().getTraces().size()).isEqualTo(spans.size());
}
 

private ProcessRecordsInput createTestData(int count) {
  List<Record> records = new ArrayList<>();

  Span[] spans = Arrays.copyOfRange(TestObjects.LOTS_OF_SPANS, 0, count);

  Arrays.stream(spans)
      .map(s -> ByteBuffer.wrap(SpanBytesEncoder.THRIFT.encodeList(Collections.singletonList(s))))
      .map(b -> new Record().withData(b))
      .forEach(records::add);

  return new ProcessRecordsInput().withRecords(records);
}
 

@Override
public void processRecords(ProcessRecordsInput processRecordsInput) {
    for (Record record : processRecordsInput.getRecords()) {
        String data = new String(record.getData().array(), Charset.forName("UTF-8"));
        System.out.println(data);
        if (record instanceof RecordAdapter) {
            com.amazonaws.services.dynamodbv2.model.Record streamRecord = ((RecordAdapter) record)
                    .getInternalObject();

            switch (streamRecord.getEventName()) {
                case "INSERT":
                case "MODIFY":
                    StreamsAdapterDemoHelper.putItem(dynamoDBClient, tableName,
                                                     streamRecord.getDynamodb().getNewImage());
                    break;
                case "REMOVE":
                    StreamsAdapterDemoHelper.deleteItem(dynamoDBClient, tableName,
                                                        streamRecord.getDynamodb().getKeys().get("Id").getN());
            }
        }
        checkpointCounter += 1;
        if (checkpointCounter % 10 == 0) {
            try {
                processRecordsInput.getCheckpointer().checkpoint();
            }
            catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
    
}
 

/**
 * Process data records. The Amazon Kinesis Client Library will invoke this method to deliver data records to the
 * application. Upon fail over, the new instance will get records with sequence number greater than the checkpoint
 * position for each partition key.
 *
 * @param processRecordsInput Provides the records to be processed as well as information and capabilities related
 *        to them (eg checkpointing).
 */
@Override
public void processRecords(ProcessRecordsInput processRecordsInput) {
  // KCL does not send any records to the processor that was shutdown.
  Validate.isTrue(!shutdownRequested,
      String.format("KCL returned records after shutdown is called on the processor %s.", this));
  // KCL aways gives reference to the same checkpointer instance for a given processor instance.
  checkpointer = processRecordsInput.getCheckpointer();
  List<Record> records = processRecordsInput.getRecords();
  // Empty records are expected when KCL config has CallProcessRecordsEvenForEmptyRecordList set to true.
  if (!records.isEmpty()) {
    lastProcessedRecordSeqNumber = new ExtendedSequenceNumber(records.get(records.size() - 1).getSequenceNumber());
    listener.onReceiveRecords(ssp, records, processRecordsInput.getMillisBehindLatest());
  }
}
 

protected IRecordProcessor createStreamProcessor() {
	return new IRecordProcessor() {

		@Override
		public void initialize(InitializationInput initializationInput) {
		}

		public List<Record> extractDynamoStreamRecords(List<com.amazonaws.services.kinesis.model.Record> kinesisRecords) {
			List<Record> dynamoRecords = new ArrayList<>(kinesisRecords.size());

			for(com.amazonaws.services.kinesis.model.Record kinesisRecord : kinesisRecords) {
				if (kinesisRecord instanceof RecordAdapter) {
					Record dynamoRecord = ((RecordAdapter) kinesisRecord).getInternalObject();
					dynamoRecords.add(dynamoRecord);
				}
			}

			return dynamoRecords;
		}

		@Override
		public void processRecords(ProcessRecordsInput processRecordsInput) {
			List<Record> records = extractDynamoStreamRecords(processRecordsInput.getRecords());

			DynamoDBTableReplicator.this.processRecords(records);

			checkpoint(processRecordsInput.getCheckpointer());
		}

		@Override
		public void shutdown(ShutdownInput shutdownInput) {
			if (shutdownInput.getShutdownReason() == ShutdownReason.TERMINATE) {
				checkpoint(shutdownInput.getCheckpointer());
			}
		}

		void checkpoint(IRecordProcessorCheckpointer checkpointer) {
			try {
				checkpointer.checkpoint();
			} catch (KinesisClientLibDependencyException|InvalidStateException|ThrottlingException|ShutdownException e) {
				LOG.warn(e);
			}
		}
	};
}