com.amazonaws.services.lambda.runtime.events.KinesisEvent Java Examples

/**
    * @see com.amazonaws.services.lambda.runtime.RequestHandler#handleRequest(java.lang.Object, com.amazonaws.services.lambda.runtime.Context)
    */
   public Void handleRequestWithLists(KinesisEvent event, Context context) {
	LambdaLogger logger = context.getLogger();
	logger.log("Received " + event.getRecords().size() + " raw Event Records.");

	try {
		// process the user records with an anonymous record processor
		// instance
		RecordDeaggregator.processRecords(event.getRecords(), new KinesisUserRecordProcessor() {
			public Void process(List<UserRecord> userRecords) {
				for (UserRecord userRecord : userRecords) {
					// Your User Record Processing Code Here!
					logger.log(new String(userRecord.getData().array()));
				}

				return null;
			}
		});
	} catch (Exception e) {
		logger.log(e.getMessage());
	}

	return null;
}
 

public void kinesisHandler(KinesisEvent event, Context context){
    logger = context.getLogger();
    setup();
    for(KinesisEvent.KinesisEventRecord rec : event.getRecords()) {
        logger.log("Got message ");
        String msg = new String(rec.getKinesis().getData().array())+"\n";
        Record deliveryStreamRecord = new Record().withData(ByteBuffer.wrap(msg.getBytes()));

        PutRecordRequest putRecordRequest = new PutRecordRequest()
                .withDeliveryStreamName(deliveryStreamName)
                .withRecord(deliveryStreamRecord);

        logger.log("Putting message");
        firehoseClient.putRecord(putRecordRequest);
        logger.log("Successful Put");
    }
}
 

@Test
public void testAggregatedRecord() throws Exception {
	// create a new KinesisEvent.Record from the aggregated data
	KinesisEvent.Record r = new KinesisEvent.Record();
	r.setPartitionKey(aggregated.getPartitionKey());
	r.setApproximateArrivalTimestamp(new Date(System.currentTimeMillis()));
	r.setData(ByteBuffer.wrap(aggregated.toRecordBytes()));
	r.setKinesisSchemaVersion("1.0");
	KinesisEventRecord ker = new KinesisEventRecord();
	ker.setKinesis(r);

	// deaggregate the record
	List<KinesisClientRecord> userRecords = deaggregator.deaggregate(Arrays.asList(ker));

	assertEquals("Deaggregated Count Matches", aggregated.getNumUserRecords(), userRecords.size());
	verifyOneToOneMapping(userRecords);
}
 

@Override
public Void handleRequest(KinesisEvent event, Context context) {
	LambdaLogger logger = context.getLogger();

	// extract the records from the event
	List<KinesisEventRecord> records = event.getRecords();

	logger.log(String.format("Recieved %s Raw Records", records.size()));

	try {
		// now deaggregate the message contents
		List<KinesisClientRecord> deaggregated = new RecordDeaggregator<KinesisEventRecord>().deaggregate(records);
		logger.log(String.format("Received %s Deaggregated User Records", deaggregated.size()));

		deaggregated.stream().forEachOrdered(rec -> {
			logger.log(rec.partitionKey());
		});
	} catch (Exception e) {
		logger.log(e.getMessage());
	}

	return null;
}
 

/**
    * @see com.amazonaws.services.lambda.runtime.RequestHandler#handleRequest(java.lang.Object, com.amazonaws.services.lambda.runtime.Context)
    */
   public Void handleRequestBulkList(KinesisEvent event, Context context) {
	LambdaLogger logger = context.getLogger();
	logger.log("Received " + event.getRecords().size() + " raw Event Records.");

	try {
		List<UserRecord> userRecords = RecordDeaggregator.deaggregate(event.getRecords());
		for (UserRecord userRecord : userRecords) {
			// Your User Record Processing Code Here!
			logger.log(new String(userRecord.getData().array()));
		}
	} catch (Exception e) {
		logger.log(e.getMessage());
	}

	return null;
}
 

/**
 * Handle request.
 *
 * @param kinesisEvent  - Kinesis Event passed as input to lambda handler
 * @param context - context object
 * @return true if success, else false.
 */
public Boolean handleRequest(final KinesisEvent kinesisEvent, final Context context) {

    LambdaLogger logger = context.getLogger();
    logger.log("Received Kinesis event: " + kinesisEvent);
    logger.log("Number of records: " + kinesisEvent.getRecords().size());

    try {
        kinesisEvent.getRecords().forEach(r -> {
            final KinesisEvent.Record kr = r.getKinesis();
            logger.log("Record: " + kr.toString());
            logger.log("Data: " + StandardCharsets.UTF_8.decode(kr.getData()).toString());
        });
    } catch (final Exception e) {
        logger.log("There was an exception: " + e.getMessage());
        return false;
    }

    return true;
}
 

@Test
public void testAggregatedRecord() {
	// create a new KinesisEvent.Record from the aggregated data
	KinesisEvent.Record r = new KinesisEvent.Record();
	r.setPartitionKey(aggregated.getPartitionKey());
	r.setApproximateArrivalTimestamp(new Date(System.currentTimeMillis()));
	r.setData(ByteBuffer.wrap(aggregated.toRecordBytes()));
	r.setKinesisSchemaVersion("1.0");
	KinesisEventRecord ker = new KinesisEventRecord();
	ker.setKinesis(r);

	// deaggregate the record
	List<UserRecord> userRecords = deaggregator.deaggregate(Arrays.asList(ker));

	assertEquals("Deaggregated Count Matches", aggregated.getNumUserRecords(), userRecords.size());
	verifyOneToOneMapping(userRecords);
}
 

@Override
public Void handleRequest(KinesisEvent event, Context context) {
	LambdaLogger logger = context.getLogger();

	// extract the records from the event
	List<KinesisEventRecord> records = event.getRecords();

	logger.log(String.format("Recieved %s Raw Records", records.size()));

	// now deaggregate the message contents
	List<UserRecord> deaggregated = new RecordDeaggregator<KinesisEventRecord>().deaggregate(records);
	logger.log(String.format("Received %s Deaggregated User Records", deaggregated.size()));
	
	deaggregated.stream().forEachOrdered(rec -> {
		logger.log(rec.getPartitionKey());
	});

	return null;
}
 

/**
 * Handle request for each lambda event.
 *
 * @param kinesisEvent Each kinesis event which describes the Rekognition output.
 * @param context      Lambda context
 * @return context
 */
@Override
public Context handleRequest(final KinesisEvent kinesisEvent, final Context context) {
    try {
        initialize(System.getProperty("KVSStreamName"), Regions.fromName(System.getenv("AWS_REGION")));
        loadProducerJNI(context);

        final List<Record> records = kinesisEvent.getRecords()
                .stream()
                .map(KinesisEvent.KinesisEventRecord::getKinesis)
                .collect(Collectors.toList());
        processRecordsWithRetries(records);
        processRekognizedOutputs();

    } catch (final Exception e) {
        log.error("Unable to process lambda request !. Exiting... ", e);
    }
    return context;
}
 

public void handler(KinesisEvent event, Context context) throws HandlerException {
  if (!initialized) {
    init(context);
  }

  KinesisHandlerConfig handlerConfig = (KinesisHandlerConfig) this.config.getHandlerConfig();
  this.recordIterator = new KinesisEventIterator(new LambdaContext(context),
          event.getRecords(),
          handlerConfig.getAddShardIdToPartitions(),
          handlerConfig.getDecompress(),
          handlerConfig.getBufferSize());

  /*
   * Get processors based on the source stream ARN
   */
  KinesisEventRecord firstRecord = event.getRecords().get(0);
  this.source = SourceUtils.getSource(firstRecord.getEventSourceARN(), sources);

  super.process(context);
}
 

@Bean
public Function<KinesisEvent, String> inputKinesisEvent() {
	return v -> {
		System.out.println("Received: " + v);
		return v.toString();
	};
}
 

@Override
protected Object convertEvent(KinesisEvent event) {
	List<E> payloads = deserializePayloads(event.getRecords());

	if (getInspector().isMessage(function())) {
		return wrapInMessages(payloads);
	}
	else {
		return payloads;
	}
}
 

private static KinesisEvent asAggregatedKinesisEvent(List<?> payloads) {
	RecordAggregator aggregator = new RecordAggregator();

	payloads.stream().map(SpringBootKinesisEventHandlerTests::asJsonByteBuffer)
			.forEach(buffer -> {
				try {
					aggregator.addUserRecord("fakePartitionKey", buffer.array());
				}
				catch (Exception e) {
					fail("Creating aggregated record failed");
				}
			});

	AggRecord aggRecord = aggregator.clearAndGet();

	KinesisEvent.Record record = new KinesisEvent.Record();
	record.setData(ByteBuffer.wrap(aggRecord.toRecordBytes()));

	KinesisEvent.KinesisEventRecord wrappingRecord = new KinesisEvent.KinesisEventRecord();
	wrappingRecord.setKinesis(record);
	wrappingRecord.setEventVersion("1.0");

	KinesisEvent event = new KinesisEvent();
	event.setRecords(singletonList(wrappingRecord));

	return event;
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
	aggregator = new RecordAggregator();

	recordList = new LinkedList<>();

	// create 10 random records for testing
	for (int i = 0; i < c; i++) {
		// create trackable id
		String id = UUID.randomUUID().toString();

		// create a kinesis model record
		byte[] data = RandomStringUtils.randomAlphabetic(20).getBytes();

		KinesisEvent.Record r = new KinesisEvent.Record();
		r.withPartitionKey(id).withApproximateArrivalTimestamp(new Date(System.currentTimeMillis()))
				.withData(ByteBuffer.wrap(data));
		KinesisEventRecord ker = new KinesisEventRecord();
		ker.setKinesis(r);
		recordList.add(ker);

		// add the record to the check set
		checkset.put(id, ker);

		// add the record to the aggregated AggRecord // create an aggregated set of
		aggregator.addUserRecord(id, data);
	}

	// get the aggregated data
	aggregated = aggregator.clearAndGet();
	assertEquals("Aggregated Record Count Correct", aggregated.getNumUserRecords(), c);
}
 

@Test
public void testSerializeDateIndexName() throws UnsupportedEncodingException, IOException {
  ElasticSearchTransportSerializer serializer =
      new ElasticSearchTransportSerializer(false, "event", "log-", "yyyy-MM-dd", false,
          "_routing");

  KinesisEvent kevent = TestUtils.createEvent(this.getClass(), "basic_event.json");
  String payload = new String(kevent.getRecords().get(0).getKinesis().getData().array());
  InternalEvent record = new DummyEvent(payload, 1478737790000l);

  String actual = new String(serializer.serialize(record));
  String expected = TestUtils.getResourceString(this.getClass(), "datetime_output.txt");
  assertEquals(expected, actual);
}
 

public static software.amazon.awssdk.services.kinesis.model.Record convertOne(KinesisEventRecord record) {
	KinesisEvent.Record r = record.getKinesis();
	software.amazon.awssdk.services.kinesis.model.Record out = software.amazon.awssdk.services.kinesis.model.Record
			.builder().partitionKey(r.getPartitionKey()).encryptionType(r.getEncryptionType())
			.approximateArrivalTimestamp(r.getApproximateArrivalTimestamp().toInstant())
			.sequenceNumber(r.getSequenceNumber()).data(SdkBytes.fromByteBuffer(r.getData())).build();

	return out;
}
 

private Record convertOne(KinesisEventRecord record) {
	KinesisEvent.Record r = record.getKinesis();
	Record out = Record.builder().partitionKey(r.getPartitionKey()).encryptionType(r.getEncryptionType())
			.approximateArrivalTimestamp(r.getApproximateArrivalTimestamp().toInstant())
			.sequenceNumber(r.getSequenceNumber()).data(SdkBytes.fromByteBuffer(r.getData())).build();

	return out;

}
 

@Test
public void functionBeanHandlesAggregatedKinesisEvent() throws Exception {
	this.handler = new SpringBootKinesisEventHandler<>(FunctionConfig.class);

	List<Foo> events = asList(new Foo("foo"), new Foo("bar"), new Foo("baz"));
	KinesisEvent aggregatedEvent = asAggregatedKinesisEvent(events);

	List<Bar> output = this.handler.handleRequest(aggregatedEvent, null);

	assertThat(output).containsExactly(new Bar("FOO"), new Bar("BAR"),
			new Bar("BAZ"));
}
 

/**
 * @see com.amazonaws.services.lambda.runtime.RequestHandler#handleRequest(java.lang.Object, com.amazonaws.services.lambda.runtime.Context)
 */
   public Void handleRequest(KinesisEvent event, Context context) {
	LambdaLogger logger = context.getLogger();
	logger.log("Received " + event.getRecords().size() + " raw Event Records.");

	// Stream the User Records from the Lambda Event
	RecordDeaggregator.stream(event.getRecords().stream(), userRecord -> {
		// Your User Record Processing Code Here!
		logger.log(new String(userRecord.getData().array()));
	});

	return null;
}
 

@Override
public KinesisEvent getTestEvent() throws Exception {
    KinesisEvent kinesisEvent = TestUtils.createEvent(this.getClass(),
            "basic_input.json",
            "arn:aws:kinesis:us-east-1:2341:stream/test-events-stream");
    kinesisEvent.getRecords().get(0).getKinesis().setData(getBase64DecodedGzipFile());
    return kinesisEvent;
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
	aggregator = new RecordAggregator();

	recordList = new LinkedList<>();

	// create 10 random records for testing
	for (int i = 0; i < c; i++) {
		// create trackable id
		String id = UUID.randomUUID().toString();

		// create a kinesis model record
		byte[] data = RandomStringUtils.randomAlphabetic(20).getBytes();

		KinesisEvent.Record r = new KinesisEvent.Record();
		r.withPartitionKey(id).withApproximateArrivalTimestamp(new Date(System.currentTimeMillis()))
				.withData(ByteBuffer.wrap(data));
		KinesisEventRecord ker = new KinesisEventRecord();
		ker.setKinesis(r);
		recordList.add(ker);

		// add the record to the check set
		checkset.put(id, ker);

		// add the record to the aggregated AggRecord // create an aggregated set of
		aggregator.addUserRecord(id, data);
	}

	// get the aggregated data
	aggregated = aggregator.clearAndGet();
	assertEquals("Aggregated Record Count Correct", aggregated.getNumUserRecords(), c);
}
 

private com.amazonaws.services.kinesis.model.Record convertOne(KinesisEventRecord record) {
	KinesisEvent.Record r = record.getKinesis();
	com.amazonaws.services.kinesis.model.Record out = new com.amazonaws.services.kinesis.model.Record()
			.withPartitionKey(r.getPartitionKey()).withEncryptionType(r.getEncryptionType())
			.withApproximateArrivalTimestamp(r.getApproximateArrivalTimestamp())
			.withSequenceNumber(r.getSequenceNumber()).withData(r.getData());

	return out;

}
 

public static KinesisEvent createEvent(Class clazz, String resource, String eventSourceArn) throws IOException {
  /*
   * Create a kinesis record from a sample JSON file
   */
  String json =
      IOUtils.toString(new InputStreamReader(clazz.getResourceAsStream(resource), "UTF-8"));

  Date approximateArrivalTimestamp = new Date();
  approximateArrivalTimestamp.setTime(1478737790000l);

  Record rec = new Record();
  rec.withPartitionKey("1").withSequenceNumber("2").withData(ByteBuffer.wrap(json.getBytes()))
      .withApproximateArrivalTimestamp(approximateArrivalTimestamp);

  /*
   * Create a KinesisEventRecord and add single Record
   */
  KinesisEventRecord krecord = new KinesisEventRecord();
  krecord.setKinesis(rec);
  krecord.setEventSourceARN(eventSourceArn);
  krecord.setEventID("shardId-000000000000:1234");

  /*
   * Add single KinesisEventRecord to a KinesisEvent
   */
  KinesisEvent kevent = new KinesisEvent();
  List<KinesisEventRecord> events = new ArrayList<KinesisEventRecord>(1);
  events.add(krecord);
  kevent.setRecords(events);

  return kevent;
}
 

@Bean
public Function<Message<KinesisEvent>, String> inputKinesisEventAsMessage() {
	return v -> {
		System.out.println("Received: " + v);
		return v.toString();
	};
}
 

private static KinesisEvent asKinesisEvent(List<Object> payloads) {
	KinesisEvent kinesisEvent = new KinesisEvent();

	List<KinesisEvent.KinesisEventRecord> kinesisEventRecords = new ArrayList<>();

	for (Object payload : payloads) {
		KinesisEvent.Record record = new KinesisEvent.Record();
		record.setData(asJsonByteBuffer(payload));

		KinesisEvent.KinesisEventRecord kinesisEventRecord = new KinesisEvent.KinesisEventRecord();
		kinesisEventRecord.setKinesis(record);

		kinesisEventRecords.add(kinesisEventRecord);
	}

	kinesisEvent.setRecords(kinesisEventRecords);

	return kinesisEvent;
}
 

@Override
protected List<KinesisEventRecord> convertEvent(KinesisEvent event) {
	// TODO: maybe convert to List<Message>
	return event.getRecords();
}
 

@Override
public KinesisEvent getTestEvent() throws Exception {
  return TestUtils.createEvent(this.getClass(), "basic_input.json");
}
 

private List<E> deserializePayloads(List<KinesisEvent.KinesisEventRecord> records) {
	return RecordDeaggregator.deaggregate(records).stream()
			.map(this::deserializeUserRecord).collect(toList());
}
 

@SuppressWarnings("unchecked")
@Override
public List<O> handleRequest(KinesisEvent event, Context context) {
	return (List<O>) super.handleRequest(event, context);
}
 

@Override
public KinesisEvent getTestEvent() throws Exception {
  return TestUtils.createEvent(this.getClass(),
          "basic_input.json",
          "arn:aws:kinesis:us-east-1:1234:stream/test-events-stream");
}