com.amazonaws.services.s3.event.S3EventNotification Java Examples

@Override
public S3EventNotification getTestEvent() throws Exception {
  /*
   * Upload a test resoruce to the mock S3
   */
  String payload = IOUtils.toString(
      new InputStreamReader(this.getClass().getResourceAsStream("basic_input.log"), "UTF-8"));
  this.client.putObject(S3_BUCKET, "basic_input.log", payload);

  /*
   * Create a S3EventNotification event
   */
  S3ObjectEntity objEntity = new S3ObjectEntity("basic_input.log", 1L, null, null);
  S3BucketEntity bucketEntity = new S3BucketEntity(S3_BUCKET, null, null);
  S3Entity entity = new S3Entity(null, bucketEntity, objEntity, null);

  S3EventNotificationRecord rec = new S3EventNotificationRecord(null, null, null,
      "1970-01-01T00:00:00.000Z", null, null, null, entity, null);

  List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(2);
  notifications.add(rec);

  return new S3EventNotification(notifications);
}
 

@Test
public void testS3MessageS3FileNoExists() throws Exception
{
    setLogLevel(UploadDownloadHelperServiceImpl.class, LogLevel.OFF);

    uploadDownloadServiceTestHelper.createDatabaseEntitiesForUploadDownloadTesting();

    UploadSingleInitiationResponse resultUploadSingleInitiationResponse = uploadDownloadService.initiateUploadSingle(uploadDownloadServiceTestHelper
        .createUploadSingleInitiationRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, NAMESPACE, BDEF_NAME_2,
            FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, FORMAT_VERSION_2, TARGET_S3_KEY));

    String filePath = resultUploadSingleInitiationResponse.getSourceBusinessObjectData().getStorageUnits().get(0).getStorageFiles().get(0).getFilePath();

    S3Entity s3Entity = new S3Entity(null, null, new S3ObjectEntity(filePath, 0L, null, null), null);

    List<S3EventNotificationRecord> records = new ArrayList<>();
    records.add(new S3EventNotificationRecord(null, null, null, null, null, null, null, s3Entity, null));

    S3EventNotification s3EventNotification = new S3EventNotification(records);

    setLogLevel(UploadDownloadServiceImpl.class, LogLevel.OFF);
    setLogLevel(HerdJmsMessageListener.class, LogLevel.OFF);

    // Try to process an S3 JMS message, when source S3 file does not exist.
    herdJmsMessageListener.processMessage(jsonHelper.objectToJson(s3EventNotification), null);
}
 

private S3EventNotification getTestEvent(String bucket, boolean doPut) throws Exception {
  /*
   * Upload a test resoruce to the mock S3
   */
  if (doPut) {
    String payload = IOUtils.toString(
        new InputStreamReader(this.getClass().getResourceAsStream("basic_input.log"), "UTF-8"));
    this.client.putObject(bucket, "basic_input.log", payload);
  }

  /*
   * Create a S3EventNotification event
   */
  S3ObjectEntity objEntity = new S3ObjectEntity("basic_input.log", 1L, null, null);
  S3BucketEntity bucketEntity = new S3BucketEntity(bucket, null, null);
  S3Entity entity = new S3Entity(null, bucketEntity, objEntity, null);

  S3EventNotificationRecord rec = new S3EventNotificationRecord(null, null, null,
      "1970-01-01T00:00:00.000Z", null, null, null, entity, null);

  List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(2);
  notifications.add(rec);

  return new S3EventNotification(notifications);
}
 

public static S3EventNotification getS3Notification(String key, String bucket, long size) {
  S3ObjectEntity objEntity = new S3ObjectEntity(key, size, null, null);
  S3BucketEntity bucketEntity = new S3BucketEntity(bucket, null, null);
  S3Entity entity = new S3Entity(null, bucketEntity, objEntity, null);

  String timestamp = formatter.print(System.currentTimeMillis());
  S3EventNotificationRecord rec =
      new S3EventNotificationRecord(null, null, null, timestamp, null, null, null, entity, null);

  List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(1);
  notifications.add(rec);

  return new S3EventNotification(notifications);
}
 

@Override
public Void handleRequest(S3Event s3Event, Context context) {

    Collection<Partition> partitionsToRemove = new HashSet<>();
    TableService tableService = new TableService();

    for (S3EventNotification.S3EventNotificationRecord record : s3Event.getRecords()) {

        String bucket = record.getS3().getBucket().getName();
        String key = record.getS3().getObject().getKey();

        System.out.printf("S3 event [Event: %s, Bucket: %s, Key: %s]%n", record.getEventName(), bucket, key);

        S3Object s3Object = new S3Object(bucket, key);

        if (s3Object.hasDateTimeKey()) {
            partitionsToRemove.add(partitionConfig.createPartitionFor(s3Object));
        }
    }

    if (!partitionsToRemove.isEmpty()) {
        tableService.removePartitions(
                partitionConfig.tableName(),
                partitionsToRemove.stream().map(Partition::spec).collect(Collectors.toList()));
    }

    return null;
}
 

@Override
public Void handleRequest(S3Event s3Event, Context context) {

    Collection<Partition> requiredPartitions = new HashSet<>();
    TableService tableService = new TableService();

    for (S3EventNotification.S3EventNotificationRecord record : s3Event.getRecords()) {

        String bucket = record.getS3().getBucket().getName();
        String key = record.getS3().getObject().getKey();

        System.out.printf("S3 event [Event: %s, Bucket: %s, Key: %s]%n", record.getEventName(), bucket, key);

        S3Object s3Object = new S3Object(bucket, key);

        if (s3Object.hasDateTimeKey()) {
            requiredPartitions.add(partitionConfig.createPartitionFor(s3Object));
        }
    }

    if (!requiredPartitions.isEmpty()) {
        Collection<Partition> missingPartitions = determineMissingPartitions(
                partitionConfig.tableName(),
                requiredPartitions,
                tableService);
        tableService.addPartitions(partitionConfig.tableName(), missingPartitions);
    }

    return null;
}
 

@Test
public void testS3MessageNoKey() throws Exception
{
    S3Entity s3Entity = new S3Entity(null, null, new S3ObjectEntity("key_does_not_exist", 0L, null, null), null);
    List<S3EventNotificationRecord> records = new ArrayList<>();
    records.add(new S3EventNotificationRecord(null, null, null, null, null, null, null, s3Entity, null));

    S3EventNotification s3EventNotification = new S3EventNotification(records);

    setLogLevel(UploadDownloadServiceImpl.class, LogLevel.OFF);
    setLogLevel(HerdJmsMessageListener.class, LogLevel.OFF);

    herdJmsMessageListener.processMessage(jsonHelper.objectToJson(s3EventNotification), null);
}
 

@Test
public void testS3MessageWithWrongFormat() throws Exception
{
    // Create and persist database entities required for testing.
    businessObjectDefinitionServiceTestHelper.createDatabaseEntitiesForBusinessObjectDefinitionTesting();

    storageDaoTestHelper.createStorageEntity(StorageEntity.SAMPLE_DATA_FILE_STORAGE,
        Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME)));

    // Create a business object definition.
    BusinessObjectDefinitionCreateRequest request =
        new BusinessObjectDefinitionCreateRequest(NAMESPACE, BDEF_NAME, DATA_PROVIDER_NAME, BDEF_DESCRIPTION, BDEF_DISPLAY_NAME,
            businessObjectDefinitionServiceTestHelper.getNewAttributes());
    businessObjectDefinitionService.createBusinessObjectDefinition(request);
    String fileName = "test1.csv";
    String filePath = NAMESPACE + "/" + BDEF_NAME + fileName;
    long fileSize = 1024L;
    S3Entity s3Entity = new S3Entity(null, null, new S3ObjectEntity(filePath, fileSize, null, null), null);

    List<S3EventNotificationRecord> records = new ArrayList<>();
    records.add(new S3EventNotificationRecord(null, null, null, null, null, null, null, s3Entity, null));

    S3EventNotification s3EventNotification = new S3EventNotification(records);

    try
    {
        sampleDataJmsMessageListener.processMessage(jsonHelper.objectToJson(s3EventNotification), null);
    }
    catch (IllegalArgumentException ex)
    {
        //this exception should be caught inside the processMessage method
        fail();
    }
}
 

/**
 * Process the message as S3 notification.
 *
 * @param payload the JMS message payload.
 *
 * @return boolean whether message was processed.
 */
private boolean processS3Notification(String payload)
{
    boolean messageProcessed = false;

    try
    {
        // Process messages coming from S3 bucket.
        S3EventNotification s3EventNotification = S3EventNotification.parseJson(payload);
        String objectKey = URLDecoder.decode(s3EventNotification.getRecords().get(0).getS3().getObject().getKey(), CharEncoding.UTF_8);

        // Perform the complete upload single file.
        CompleteUploadSingleMessageResult completeUploadSingleMessageResult = uploadDownloadService.performCompleteUploadSingleMessage(objectKey);

        if (LOGGER.isDebugEnabled())
        {
            LOGGER.debug("completeUploadSingleMessageResult={}", jsonHelper.objectToJson(completeUploadSingleMessageResult));
        }

        messageProcessed = true;
    }
    catch (RuntimeException | UnsupportedEncodingException e)
    {
        // The logging is set to DEBUG level, since the method is expected to fail when message is not of the expected type.
        LOGGER.debug("Failed to process message from the JMS queue for an S3 notification. jmsQueueName=\"{}\" jmsMessagePayload={}",
            HerdJmsDestinationResolver.SQS_DESTINATION_HERD_INCOMING, payload, e);
    }

    return messageProcessed;
}
 

@Test
public void testSourceRegex() throws Throwable {
  BaseHandler.CONFIG_FILE = "/com/nextdoor/bender/handler/config_s3_source.json";

  TestContext ctx = new TestContext();
  ctx.setFunctionName("unittest");
  ctx.setInvokedFunctionArn("arn:aws:lambda:us-east-1:123:function:test-function:staging");

  BaseHandler<S3EventNotification> handler = (BaseHandler) getHandler();
  handler.init(ctx);

  handler.handler(getTestEvent(), ctx);
  assertEquals(1, DummyTransportHelper.BufferedTransporter.output.size());
}
 

@Test
public void testSourceRegexFail() throws Throwable {
  BaseHandler.CONFIG_FILE = "/com/nextdoor/bender/handler/config_s3_source.json";

  TestContext ctx = new TestContext();
  ctx.setFunctionName("unittest");
  ctx.setInvokedFunctionArn("arn:aws:lambda:us-east-1:123:function:test-function:staging");

  BaseHandler<S3EventNotification> handler = (BaseHandler) getHandler();
  handler.init(ctx);

  handler.handler(getTestEvent("foo", false), ctx);
  assertEquals(0, DummyTransportHelper.BufferedTransporter.output.size());
}
 

public void handler(S3EventNotification event, Context context) throws HandlerException {
  if (!initialized) {
    init(context);
    S3HandlerConfig handlerConfig = (S3HandlerConfig) this.config.getHandlerConfig();
    this.logTrigger = handlerConfig.getLogS3Trigger();
  }

  if (this.logTrigger) {
    logger.info("trigger: " + gson.toJson(event));
  }

  /*
   * Validate the S3 file matches the regex
   */
  List<S3EventNotificationRecord> toProcess =
      new ArrayList<S3EventNotificationRecord>(event.getRecords());
  for (S3EventNotificationRecord record : event.getRecords()) {
    String s3Path = String.format("s3://%s/%s", record.getS3().getBucket().getName(),
        record.getS3().getObject().getKey());
    try {
      this.source = SourceUtils.getSource(s3Path, this.sources);
    } catch (SourceNotFoundException e) {
      logger.warn("Skipping processing " + s3Path);
      toProcess.remove(record);
    }
  }

  if (toProcess.size() == 0) {
    logger.warn("Nothing to process");
    return;
  }

  this.recordIterator =
      new S3EventIterator(new LambdaContext(context), toProcess, s3ClientFactory);

  super.process(context);
}
 

@Override
public Handler<S3EventNotification> getHandler() {
  S3Handler handler = new S3Handler();
  handler.s3ClientFactory = this.clientFactory;

  return handler;
}
 

@Override
public SNSEvent getTestEvent() throws Exception {
  /*
   * Upload a test resoruce to the mock S3
   */
  String payload = IOUtils.toString(
      new InputStreamReader(this.getClass().getResourceAsStream("basic_input.log"), "UTF-8"));
  this.client.putObject(S3_BUCKET, "basic_input.log", payload);

  /*
   * Create a S3EventNotification event
   */
  S3ObjectEntity objEntity = new S3ObjectEntity("basic_input.log", 1L, null, null);
  S3BucketEntity bucketEntity = new S3BucketEntity(S3_BUCKET, null, null);
  S3Entity entity = new S3Entity(null, bucketEntity, objEntity, null);

  S3EventNotificationRecord rec = new S3EventNotificationRecord(null, null, null,
      "1970-01-01T00:00:00.000Z", null, null, null, entity, null);

  List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(2);
  notifications.add(rec);

  /*
   * Wrap as an SNS Event
   */
  S3EventNotification event = new S3EventNotification(notifications);
  SNSEvent.SNS sns = new SNSEvent.SNS();
  sns.setMessage(event.toJson());

  SNSEvent snsEvent = new SNSEvent();

  ArrayList<SNSRecord> snsRecords = new ArrayList<SNSRecord>(1);
  SNSRecord snsRecord = new SNSRecord();
  snsRecord.setEventSource("aws:sns");
  snsRecord.setEventVersion("1.0");
  snsRecord.setEventSubscriptionArn("arn");

  snsRecord.setSns(sns);
  snsRecords.add(snsRecord);
  snsEvent.setRecords(snsRecords);

  return snsEvent;
}
 

protected static void invokeS3Handler(String source_file) throws HandlerException {
  /*
   * https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html
   * https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
   */
  String awsRegion = "us-east-1";
  String eventName = "s3:ObjectCreated:Put";
  String eventSource = "aws:s3";
  String eventVersion = "2.0";
  String s3ConfigurationId = "cli-id";
  String s3SchemaVersion = "1.0";

  S3BucketEntity s3BucketEntity = null;
  S3ObjectEntity s3ObjectEntity = null;

  /*
   * Make sure the URL was submitted properly
   *
   * Split the s3://bucket/object path into an S3BucketEntity and S3ObjectEntity object
   */
  try {
    AmazonS3URI s3URI = new AmazonS3URI(source_file);
    s3BucketEntity = new S3BucketEntity(s3URI.getBucket(), null, null);
    s3ObjectEntity = new S3ObjectEntity(s3URI.getKey(), 1L, null, null);
  } catch (IllegalArgumentException e) {
    logger.error(
        "Invalid source_file URL supplied (" + source_file + "): " + e);
    System.exit(1);
  }

  /*
   * Override the AWS Region if its supplied
   */
  if (System.getenv("AWS_REGION") != null) {
    awsRegion = System.getenv("AWS_REGION");
  }

  /*
   * Set the arrival timestamp as the function run time.
   */
  DateTime eventTime = new DateTime().toDateTime();

  /*
   * Generate our context/handler objects.. we'll be populating them shortly.
   */
  TestContext ctx = getContext();
  S3Handler handler = new S3Handler();

  /*
   * Create a S3EventNotification event
   */
  S3Entity s3Entity = new S3Entity(s3ConfigurationId, s3BucketEntity, s3ObjectEntity,
      s3SchemaVersion);
  S3EventNotificationRecord rec = new S3EventNotificationRecord(awsRegion, eventName, eventSource,
      eventTime.toString(), eventVersion, null, null, s3Entity, null);
  List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(2);
  notifications.add(rec);
  S3EventNotification s3event = new S3EventNotification(notifications);

  /*
   * Invoke handler
   */
  handler.handler(s3event, ctx);
  handler.shutdown();
}
 

/**
 * Processes a JMS message.
 *
 * @param payload the message payload
 * @param allHeaders the JMS headers
 */
@JmsListener(id = HerdJmsDestinationResolver.SQS_DESTINATION_SAMPLE_DATA_QUEUE,
    containerFactory = "jmsListenerContainerFactory",
    destination = HerdJmsDestinationResolver.SQS_DESTINATION_SAMPLE_DATA_QUEUE)
public void processMessage(String payload, @Headers Map<Object, Object> allHeaders)
{
    LOGGER.info("Message received from the JMS queue. jmsQueueName=\"{}\" jmsMessageHeaders=\"{}\" jmsMessagePayload={}",
        HerdJmsDestinationResolver.SQS_DESTINATION_SAMPLE_DATA_QUEUE, allHeaders, payload);

    try
    {
        // Process messages coming from S3 bucket.
        S3EventNotification s3EventNotification = S3EventNotification.parseJson(payload);
        String objectKey = URLDecoder.decode(s3EventNotification.getRecords().get(0).getS3().getObject().getKey(), CharEncoding.UTF_8);
        long fileSize = s3EventNotification.getRecords().get(0).getS3().getObject().getSizeAsLong();
        // parse the objectKey, it should be in the format of namespace/businessObjectDefinitionName/fileName
        String[] objectKeyArrays = objectKey.split("/");
        Assert.isTrue(objectKeyArrays.length == 3, String.format("S3 notification message %s is not in expected format", objectKey));

        String namespace = objectKeyArrays[0];
        String businessObjectDefinitionName = objectKeyArrays[1];
        String fileName = objectKeyArrays[2];
        String path = namespace + "/" + businessObjectDefinitionName + "/";
        BusinessObjectDefinitionSampleFileUpdateDto businessObjectDefinitionSampleFileUpdateDto =
                new BusinessObjectDefinitionSampleFileUpdateDto(path, fileName, fileSize);

        String convertedNamespaece = convertS3KeyFormat(namespace);
        String convertedBusinessObjectDefinitionName = convertS3KeyFormat(businessObjectDefinitionName);

        BusinessObjectDefinitionKey businessObjectDefinitionKey =
                new BusinessObjectDefinitionKey(convertedNamespaece, convertedBusinessObjectDefinitionName);
        try
        {
            businessObjectDefinitionService.updateBusinessObjectDefinitionEntitySampleFile(businessObjectDefinitionKey,
                    businessObjectDefinitionSampleFileUpdateDto);
        }
        catch (ObjectNotFoundException ex)
        {
            LOGGER.info("Failed to find the business object definition, next try the original namespace and business oject defination name " + ex);
            // if Business object definition is not found, use the original name space and bdef name
            businessObjectDefinitionKey = new BusinessObjectDefinitionKey(namespace, businessObjectDefinitionName);
            businessObjectDefinitionService.updateBusinessObjectDefinitionEntitySampleFile(businessObjectDefinitionKey,
                    businessObjectDefinitionSampleFileUpdateDto);
        }
    }
    catch (RuntimeException | IOException e)
    {
        LOGGER.error("Failed to process message from the JMS queue. jmsQueueName=\"{}\" jmsMessagePayload={}",
                HerdJmsDestinationResolver.SQS_DESTINATION_SAMPLE_DATA_QUEUE, payload, e);
    }
}
 

@Override
public S3EventNotification getTestEvent() throws Exception {
  return getTestEvent(S3_BUCKET, true);
}
 

@Test
public void testS3Message() throws Exception
{
    // Create and persist database entities required for testing.
    businessObjectDefinitionServiceTestHelper.createDatabaseEntitiesForBusinessObjectDefinitionTesting();

    storageDaoTestHelper.createStorageEntity(StorageEntity.SAMPLE_DATA_FILE_STORAGE,
        Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME)));

    // Create a business object definition.
    BusinessObjectDefinitionCreateRequest request =
        new BusinessObjectDefinitionCreateRequest(NAMESPACE, BDEF_NAME, DATA_PROVIDER_NAME, BDEF_DESCRIPTION, BDEF_DISPLAY_NAME,
            businessObjectDefinitionServiceTestHelper.getNewAttributes());
    businessObjectDefinitionService.createBusinessObjectDefinition(request);

    // Get the business object definition entity.
    BusinessObjectDefinitionEntity businessObjectDefinitionEntity =
        businessObjectDefinitionDao.getBusinessObjectDefinitionByKey(new BusinessObjectDefinitionKey(NAMESPACE, BDEF_NAME));
    assertNotNull(businessObjectDefinitionEntity);

    String fileName = "test1.csv";
    String filePath = NAMESPACE + "/" + BDEF_NAME + "/" + fileName;
    long fileSize = 1024L;
    S3Entity s3Entity = new S3Entity(null, null, new S3ObjectEntity(filePath, fileSize, null, null), null);

    List<S3EventNotificationRecord> records = new ArrayList<>();
    records.add(new S3EventNotificationRecord(null, null, null, null, null, null, null, s3Entity, null));

    S3EventNotification s3EventNotification = new S3EventNotification(records);

    sampleDataJmsMessageListener.processMessage(jsonHelper.objectToJson(s3EventNotification), null);
    BusinessObjectDefinitionKey businessObjectDefinitionKey = new BusinessObjectDefinitionKey(NAMESPACE, BDEF_NAME);
    BusinessObjectDefinition updatedBusinessObjectDefinition =
        businessObjectDefinitionService.getBusinessObjectDefinition(businessObjectDefinitionKey, false);

    List<SampleDataFile> sampleDataFiles = Arrays.asList(new SampleDataFile(NAMESPACE + "/" + BDEF_NAME + "/", fileName));

    // Validate the returned object.
    assertEquals(new BusinessObjectDefinition(updatedBusinessObjectDefinition.getId(), NAMESPACE, BDEF_NAME, DATA_PROVIDER_NAME, BDEF_DESCRIPTION,
        NO_BDEF_SHORT_DESCRIPTION, BDEF_DISPLAY_NAME, businessObjectDefinitionServiceTestHelper.getNewAttributes(), NO_DESCRIPTIVE_BUSINESS_OBJECT_FORMAT,
        sampleDataFiles, businessObjectDefinitionEntity.getCreatedBy(), businessObjectDefinitionEntity.getUpdatedBy(),
            HerdDateUtils.getXMLGregorianCalendarValue(businessObjectDefinitionEntity.getUpdatedOn()), NO_BUSINESS_OBJECT_DEFINITION_CHANGE_EVENTS),
        updatedBusinessObjectDefinition);
}
 

@Test
public void testS3MessageWithDashCharacterName() throws Exception
{
    String namespace = "testnamespace-1";
    String businessObjectDefinitionName = "testbdefname-1";

    // Create and persist database entities required for testing.
    businessObjectDefinitionServiceTestHelper.createDatabaseEntitiesForBusinessObjectDefinitionTesting(namespace, DATA_PROVIDER_NAME);

    storageDaoTestHelper.createStorageEntity(StorageEntity.SAMPLE_DATA_FILE_STORAGE,
        Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME)));

    // Create a business object definition.
    BusinessObjectDefinitionCreateRequest request =
        new BusinessObjectDefinitionCreateRequest(namespace, businessObjectDefinitionName, DATA_PROVIDER_NAME, BDEF_DESCRIPTION, BDEF_DISPLAY_NAME,
            businessObjectDefinitionServiceTestHelper.getNewAttributes());
    businessObjectDefinitionService.createBusinessObjectDefinition(request);

    // Get the business object definition entity.
    BusinessObjectDefinitionEntity businessObjectDefinitionEntity =
        businessObjectDefinitionDao.getBusinessObjectDefinitionByKey(new BusinessObjectDefinitionKey(namespace, businessObjectDefinitionName));
    assertNotNull(businessObjectDefinitionEntity);

    String fileName = "test1.csv";
    String filePath = namespace + "/" + businessObjectDefinitionName + "/" + fileName;
    long fileSize = 1024L;
    S3Entity s3Entity = new S3Entity(null, null, new S3ObjectEntity(filePath, fileSize, null, null), null);

    List<S3EventNotificationRecord> records = new ArrayList<>();
    records.add(new S3EventNotificationRecord(null, null, null, null, null, null, null, s3Entity, null));

    S3EventNotification s3EventNotification = new S3EventNotification(records);

    sampleDataJmsMessageListener.processMessage(jsonHelper.objectToJson(s3EventNotification), null);
    BusinessObjectDefinitionKey businessObjectDefinitionKey = new BusinessObjectDefinitionKey(namespace, businessObjectDefinitionName);
    BusinessObjectDefinition updatedBusinessObjectDefinition =
        businessObjectDefinitionService.getBusinessObjectDefinition(businessObjectDefinitionKey, false);

    List<SampleDataFile> sampleDataFiles = Arrays.asList(new SampleDataFile(namespace + "/" + businessObjectDefinitionName + "/", fileName));

    // Validate the returned object.
    assertEquals(
        new BusinessObjectDefinition(updatedBusinessObjectDefinition.getId(), namespace, businessObjectDefinitionName, DATA_PROVIDER_NAME, BDEF_DESCRIPTION,
            NO_BDEF_SHORT_DESCRIPTION, BDEF_DISPLAY_NAME, businessObjectDefinitionServiceTestHelper.getNewAttributes(),
            NO_DESCRIPTIVE_BUSINESS_OBJECT_FORMAT, sampleDataFiles, businessObjectDefinitionEntity.getCreatedBy(),
            businessObjectDefinitionEntity.getUpdatedBy(), HerdDateUtils.getXMLGregorianCalendarValue(businessObjectDefinitionEntity.getUpdatedOn()),
            NO_BUSINESS_OBJECT_DEFINITION_CHANGE_EVENTS),
        updatedBusinessObjectDefinition);
}
 

@Override
public Handler<S3EventNotification> getHandler() {
  S3Handler handler = new S3Handler();
  handler.s3ClientFactory = this.clientFactory;
  return handler;
}
 

@Test
public void testS3Message() throws Exception
{
    setLogLevel(UploadDownloadHelperServiceImpl.class, LogLevel.OFF);

    uploadDownloadServiceTestHelper.createDatabaseEntitiesForUploadDownloadTesting();

    UploadSingleInitiationResponse resultUploadSingleInitiationResponse =
        uploadDownloadService.initiateUploadSingle(uploadDownloadServiceTestHelper.createUploadSingleInitiationRequest());

    String filePath = resultUploadSingleInitiationResponse.getSourceBusinessObjectData().getStorageUnits().get(0).getStorageFiles().get(0).getFilePath();

    S3Entity s3Entity = new S3Entity(null, null, new S3ObjectEntity(filePath, 0L, null, null), null);

    List<S3EventNotificationRecord> records = new ArrayList<>();
    records.add(new S3EventNotificationRecord(null, null, null, null, null, null, null, s3Entity, null));

    S3EventNotification s3EventNotification = new S3EventNotification(records);

    setLogLevel(UploadDownloadServiceImpl.class, LogLevel.OFF);
    setLogLevel(HerdJmsMessageListener.class, LogLevel.DEBUG);

    herdJmsMessageListener.processMessage(jsonHelper.objectToJson(s3EventNotification), null);
}
 

private SNSEvent getTestEvent(String bucket, boolean doPut) throws Exception {
  /*
   * Upload a test resoruce to the mock S3
   */
  if (doPut) {
    String payload = IOUtils.toString(
        new InputStreamReader(this.getClass().getResourceAsStream("basic_input.log"), "UTF-8"));
    this.client.putObject(bucket, "basic_input.log", payload);
  }
  /*
   * Create a S3EventNotification event
   */
  S3ObjectEntity objEntity = new S3ObjectEntity("basic_input.log", 1L, null, null);
  S3BucketEntity bucketEntity = new S3BucketEntity(bucket, null, null);
  S3Entity entity = new S3Entity(null, bucketEntity, objEntity, null);

  S3EventNotificationRecord rec = new S3EventNotificationRecord(null, null, null,
      "1970-01-01T00:00:00.000Z", null, null, null, entity, null);

  List<S3EventNotificationRecord> notifications = new ArrayList<S3EventNotificationRecord>(2);
  notifications.add(rec);

  /*
   * Wrap as an SNS Event
   */
  S3EventNotification event = new S3EventNotification(notifications);
  SNSEvent.SNS sns = new SNSEvent.SNS();
  sns.setMessage(event.toJson());

  SNSEvent snsEvent = new SNSEvent();

  ArrayList<SNSRecord> snsRecords = new ArrayList<SNSRecord>(1);
  SNSRecord snsRecord = new SNSRecord();
  snsRecord.setEventSource("aws:sns");
  snsRecord.setEventVersion("1.0");
  snsRecord.setEventSubscriptionArn("arn");

  snsRecord.setSns(sns);
  snsRecords.add(snsRecord);
  snsEvent.setRecords(snsRecords);

  return snsEvent;
}
 

@Override
public void handler(SNSEvent event, Context context) throws HandlerException {
  if (!initialized) {
    init(context);
    SNSS3HandlerConfig handlerConfig = (SNSS3HandlerConfig) this.config.getHandlerConfig();
    this.logTrigger = handlerConfig.getLogSnsTrigger();
  }

  this.source = this.sources.get(0);
  this.inputFiles = new ArrayList<String>(0);

  if (this.logTrigger) {
    logger.info("trigger: " + gson.toJson(event));
  }

  for (SNSRecord record : event.getRecords()) {
    /*
     * Parse SNS as a S3 notification
     */
    String json = record.getSNS().getMessage();
    S3EventNotification s3Event = S3EventNotification.parseJson(json);

    /*
     * Validate the S3 file matches the regex
     */
    List<S3EventNotificationRecord> toProcess =
        new ArrayList<S3EventNotificationRecord>(s3Event.getRecords());
    for (S3EventNotificationRecord s3Record : s3Event.getRecords()) {
      String s3Path = String.format("s3://%s/%s", s3Record.getS3().getBucket().getName(),
          s3Record.getS3().getObject().getKey());
      try {
        this.source = SourceUtils.getSource(s3Path, this.sources);
      } catch (SourceNotFoundException e) {
        logger.warn("skipping processing " + s3Path);
        toProcess.remove(s3Record);
      }
    }

    if (toProcess.size() == 0) {
      logger.warn("Nothing to process");
      return;
    }

    this.inputFiles.addAll(toProcess.stream().map(m -> {
      return m.getS3().getObject().getKey();
    }).collect(Collectors.toList()));

    this.recordIterator =
        new S3EventIterator(new LambdaContext(context), toProcess, s3ClientFactory);

    super.process(context);
  }
}