Java Code Examples for com.amazonaws.util.IOUtils#toString()

public String downloadObject(final String bucketName, final String key) {

        final S3Object object = s3client.getObject(new GetObjectRequest(bucketName, key));

        final InputStream inputStream = object.getObjectContent();

        String result = null;
        try {
            result = IOUtils.toString(inputStream);
        } catch (final IOException e) {
            log.warn("Could not download file for bucket: {}, with key: {}", bucketName, key);
        } finally {
            if (inputStream != null) {
                try {
                    inputStream.close();
                } catch (final IOException ex) {
                    log.debug("Ignore failure in closing the Closeable", ex);
                }
            }
        }

        log.info("Downloaded file for bucket: {}, with key: {}", bucketName, key);
        return result;
    }
 

@Test
public void copyOneFile() throws Exception {
  client.putObject("source", "data", inputData);

  Path sourceBaseLocation = new Path("s3://source/data");
  Path replicaLocation = new Path("s3://target/data2");
  List<Path> sourceSubLocations = new ArrayList<>();
  S3S3Copier s3s3Copier = newS3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation);
  Metrics metrics = s3s3Copier.copy();
  assertThat(metrics.getBytesReplicated(), is(7L));
  assertThat(metrics.getMetrics().get(S3S3CopierMetrics.Metrics.TOTAL_BYTES_TO_REPLICATE.name()), is(7L));
  S3Object object = client.getObject("target", "data2");
  String data = IOUtils.toString(object.getObjectContent());
  assertThat(data, is("bar foo"));
  assertThat(registry.getGauges().containsKey(RunningMetrics.S3S3_CP_BYTES_REPLICATED.name()), is(true));
}
 

@Test
public void copyOneObject() throws Exception {
  client.putObject("source", "data", inputData);

  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();
  S3S3Copier s3s3Copier = newS3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation);
  Metrics metrics = s3s3Copier.copy();
  assertThat(metrics.getBytesReplicated(), is(7L));
  assertThat(metrics.getMetrics().get(S3S3CopierMetrics.Metrics.TOTAL_BYTES_TO_REPLICATE.name()), is(7L));
  S3Object object = client.getObject("target", "data");
  String data = IOUtils.toString(object.getObjectContent());
  assertThat(data, is("bar foo"));
  assertThat(registry.getGauges().containsKey(RunningMetrics.S3S3_CP_BYTES_REPLICATED.name()), is(true));
}
 

public void writeBom(String bomPath) {
  if (googleWriteableProfileRegistry == null) {
    throw new HalException(FATAL, NO_WRITER_ENABLED);
  }

  BillOfMaterials bom;
  String bomContents;
  String version;

  try {
    bomContents = IOUtils.toString(new FileInputStream(bomPath));
    bom = relaxedObjectMapper.convertValue(yamlParser.load(bomContents), BillOfMaterials.class);
    version = bom.getVersion();
  } catch (IOException e) {
    throw new HalException(
        new ConfigProblemBuilder(FATAL, "Unable to load Bill of Materials: " + e.getMessage())
            .build());
  }

  if (version == null) {
    throw new HalException(
        new ConfigProblemBuilder(FATAL, "No version was supplied in this BOM.").build());
  }

  googleWriteableProfileRegistry.writeBom(bom.getVersion(), bomContents);
}
 

@Override
public boolean isAvailableInDatabase(String state) {
    AmazonS3 s3 = this.createS3Client();
    S3Object s3Object = getObject(s3, getKey(state));
    if (s3Object == null) {
        return false;
    }
    String millisToExpire = null;
    try {
        millisToExpire = IOUtils.toString(s3Object.getObjectContent());
        return Long.valueOf(millisToExpire) > System.currentTimeMillis();
    } catch (IOException e) {
        log.error("Failed to load a state data for state: {}", state, e);
        return false;
    } catch (NumberFormatException ne) {
        log.error("Invalid state value detected - state: {}, millisToExpire: {}", state, millisToExpire);
        return false;
    }
}
 

@Test
public void testNullGeometry() throws Exception {

  // =========== INSERT ==========
  String insertJsonFile = "/events/InsertNullGeometry.json";
  String insertResponse = invokeLambdaFromFile(insertJsonFile);
  logger.info("RAW RESPONSE: " + insertResponse);
  String insertRequest = IOUtils.toString(GSContext.class.getResourceAsStream(insertJsonFile));
  assertRead(insertRequest, insertResponse, false);
  logger.info("Preparation: Insert features");

  // =========== Validate that "geometry":null is serialized ==========
  String response = invokeLambdaFromFile("/events/GetFeaturesByIdEvent.json");
  assertTrue(response.indexOf("\"geometry\":null") > 0);
}
 

/**
 * Reads the file from S3 bucket and returns as a string.
 *
 * @return File content
 * @throws IOException
 */
public static String getFile(String bucket, String key) throws IOException {
    S3Object object = getS3Client().getObject(
            new GetObjectRequest(bucket, key));
    InputStream objectData = object.getObjectContent();
    String theString = IOUtils.toString(objectData);
    objectData.close();
    return theString;
}
 

private static String invokeLambdaFromFile(String file) throws Exception {
  InputStream jsonStream = GSContext.class.getResourceAsStream(file);
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  lambda.handleRequest(jsonStream, os, GSContext.newLocal());
  String response = IOUtils.toString(Payload.prepareInputStream(new ByteArrayInputStream(os.toByteArray())));
  logger.info("Response from lambda - {}", response);
  return response;
}
 

@Override
public void handleRequest(final InputStream input, final OutputStream output, final Context context) throws IOException {
    final String inputS = IOUtils.toString(Optional.ofNullable(input).orElse(new ByteArrayInputStream("{}".getBytes())));
    final JsonNode root = om.readTree(inputS);

    final String bucket = Optional.ofNullable(root.get(S3_BUCKET_PROPERTY)).map(JsonNode::textValue).filter(StringUtils::isNotBlank).orElse(System.getenv(S3_BUCKET_PROPERTY));
    Validate.notBlank(bucket, S3_BUCKET_PROPERTY + " hasn't been set in the request payload nor as an environment variable.");

    final String key = Optional.ofNullable(root.get(S3_KEY_PROPERTY)).map(JsonNode::textValue).filter(StringUtils::isNotBlank)
            .orElse(System.getenv(S3_KEY_PROPERTY));
    final String region = Optional.ofNullable(root.get(S3_REGION_PROPERTY)).map(JsonNode::textValue).filter(StringUtils::isNotBlank)
            .orElse(System.getenv(S3_REGION_PROPERTY));

    final AmazonS3 s3client = StringUtils.isNotBlank(region) ? AmazonS3ClientBuilder.standard().withRegion(region).build() : AmazonS3ClientBuilder.defaultClient();

    final ListObjectsRequest listRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(Optional.ofNullable(key).map(k -> k + (k.endsWith("/") ? "" : "/")).orElse(""));

    log.info("[INFO] Reading out *.yml conversation script files in folder '" + listRequest.getPrefix() + "' in bucket '" + listRequest.getBucketName() + "'");

    final List<S3ObjectSummary> conversationScripts = s3client.listObjects(listRequest).getObjectSummaries().stream()
            .filter(os -> os.getKey().toLowerCase().endsWith(".yml")).collect(Collectors.toList());

    log.info("[INFO] Found " + conversationScripts.size() + " conversation script files in bucket '" + bucket + "'");

    for (final S3ObjectSummary conversationScript : conversationScripts) {
        log.info("[INFO] Load conversation script file " + conversationScript.getKey() + " from S3 bucket " + bucket);

        AlexaClient.create(s3client.getObject(bucket, conversationScript.getKey()).getObjectContent())
                .build()
                .startScript();
    }
    output.write("{ \"OK\" }".getBytes());
}
 

public GenericApiGatewayResponse(HttpResponse httpResponse) throws IOException {
    this.httpResponse = httpResponse;
    if(httpResponse.getContent() != null) {
        this.body = IOUtils.toString(httpResponse.getContent());
    }else {
        this.body = null;
    }
}
 

@Test
public void testCrudFeatureWithHash() throws Exception {
  // =========== INSERT ==========
  String insertJsonFile = "/events/InsertFeaturesEventWithHash.json";
  String insertResponse = invokeLambdaFromFile(insertJsonFile);
  String insertRequest = IOUtils.toString(GSContext.class.getResourceAsStream(insertJsonFile));
  assertRead(insertRequest, insertResponse, true);
  logger.info("Insert feature tested successfully");

  // =========== UPDATE ==========
  FeatureCollection featureCollection = XyzSerializable.deserialize(insertResponse);
  setPUUID(featureCollection);
  String featuresList = XyzSerializable.serialize(featureCollection.getFeatures(), new TypeReference<List<Feature>>() {
  });
  String updateRequest = "{\n" +
      "    \"type\": \"ModifyFeaturesEvent\",\n" +
      "    \"space\": \"foo\",\n" +
      "    \"enableUUID\": true,\n" +
      "    \"params\": {},\n" +
      "    \"updateFeatures\": " + featuresList + "\n" +
      "}";
  updateRequest = updateRequest.replaceAll("Tesla", "Honda");
  String updateResponse = invokeLambda(updateRequest);

  FeatureCollection responseCollection = XyzSerializable.deserialize(updateResponse);
  setPUUID(responseCollection);

  assertUpdate(updateRequest, updateResponse, true);
  assertUpdate(updateRequest, updateResponse, true);
  logger.info("Update feature tested successfully");
}
 

@Test
public void copyOneObjectUsingKeys() throws Exception {
  client.putObject("source", "bar/data", inputData);

  Path sourceBaseLocation = new Path("s3://source/bar/");
  Path replicaLocation = new Path("s3://target/foo/");
  List<Path> sourceSubLocations = new ArrayList<>();
  S3S3Copier s3s3Copier = newS3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation);
  s3s3Copier.copy();
  S3Object object = client.getObject("target", "foo/data");
  String data = IOUtils.toString(object.getObjectContent());
  assertThat(data, is("bar foo"));
}
 

@Test
public void copyOneObjectPartitioned() throws Exception {
  client.putObject("source", "year=2016/data", inputData);

  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/foo/");
  List<Path> sourceSubLocations = Lists.newArrayList(new Path(sourceBaseLocation, "year=2016"));
  S3S3Copier s3s3Copier = newS3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation);
  Metrics metrics = s3s3Copier.copy();
  assertThat(metrics.getBytesReplicated(), is(7L));
  S3Object object = client.getObject("target", "foo/year=2016/data");
  String data = IOUtils.toString(object.getObjectContent());
  assertThat(data, is("bar foo"));
}
 

@Test
public void copyOneObjectPartitionedSourceBaseNested() throws Exception {
  client.putObject("source", "nested/year=2016/data", inputData);

  Path sourceBaseLocation = new Path("s3://source/nested");// no slash at the end
  Path replicaLocation = new Path("s3://target/foo/");
  List<Path> sourceSubLocations = Lists.newArrayList(new Path(sourceBaseLocation, "year=2016"));
  S3S3Copier s3s3Copier = newS3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation);
  s3s3Copier.copy();
  S3Object object = client.getObject("target", "foo/year=2016/data");
  String data = IOUtils.toString(object.getObjectContent());
  assertThat(data, is("bar foo"));
}
 

@Test
public void copyOneObjectPartitionedHandlingS3ASchemes() throws Exception {
  client.putObject("source", "year=2016/data", inputData);

  Path sourceBaseLocation = new Path("s3a://source/");
  Path replicaLocation = new Path("s3a://target/foo/");
  List<Path> sourceSubLocations = Lists.newArrayList(new Path(sourceBaseLocation, "year=2016"));
  S3S3Copier s3s3Copier = newS3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation);
  s3s3Copier.copy();
  S3Object object = client.getObject("target", "foo/year=2016/data");
  String data = IOUtils.toString(object.getObjectContent());
  assertThat(data, is("bar foo"));
}
 

@Test
public void testCrudFeatureWithoutHash() throws Exception {
  // =========== INSERT ==========
  String insertJsonFile = "/events/InsertFeaturesEvent.json";
  String insertResponse = invokeLambdaFromFile(insertJsonFile);
  logger.info("RAW RESPONSE: " + insertResponse);
  String insertRequest = IOUtils.toString(GSContext.class.getResourceAsStream(insertJsonFile));
  assertRead(insertRequest, insertResponse, false);
  logger.info("Insert feature tested successfully");

  // =========== COUNT ==========
  String countResponse = invokeLambdaFromFile("/events/CountFeaturesEvent.json");
  assertCount(insertRequest, countResponse);
  logger.info("Count feature tested successfully");

  // =========== SEARCH ==========
  String searchResponse = invokeLambdaFromFile("/events/SearchForFeaturesEvent.json");
  assertRead(insertRequest, searchResponse, false);
  logger.info("Search feature tested successfully");

  // =========== SEARCH WITH PROPERTIES ========
  String searchPropertiesResponse = invokeLambdaFromFile("/events/SearchForFeaturesByPropertiesEvent.json");
  assertRead(insertRequest, searchPropertiesResponse, false);
  logger.info("Search Properties feature tested successfully");

  // =========== UPDATE ==========
  FeatureCollection featureCollection = XyzSerializable.deserialize(insertResponse);
  String featuresList = XyzSerializable.serialize(featureCollection.getFeatures(), new TypeReference<List<Feature>>() {
  });
  String updateRequest = "{\n" +
      "    \"type\": \"ModifyFeaturesEvent\",\n" +
      "    \"space\": \"foo\",\n" +
      "    \"params\": {},\n" +
      "    \"updateFeatures\": " + featuresList + "\n" +
      "}";
  updateRequest = updateRequest.replaceAll("Tesla", "Honda");
  String updateResponse = invokeLambda(updateRequest);
  assertUpdate(updateRequest, updateResponse, false);
  logger.info("Update feature tested successfully");

  // =========== DELETE FEATURES ==========
  invokeLambdaFromFile("/events/DeleteFeaturesByTagEvent.json");
  logger.info("Delete feature tested successfully");
}
 

@SuppressWarnings({"rawtypes", "unchecked"})
private void testModifyFeatures(boolean includeOldStates) throws Exception {
  // =========== INSERT ==========
  String insertJsonFile = "/events/InsertFeaturesEvent.json";
  String insertResponse = invokeLambdaFromFile(insertJsonFile);
  logger.info("RAW RESPONSE: " + insertResponse);
  String insertRequest = IOUtils.toString(GSContext.class.getResourceAsStream(insertJsonFile));
  assertRead(insertRequest, insertResponse, false);
  final JsonPath jsonPathFeatures = JsonPath.compile("$.features");
  List<Map> originalFeatures = jsonPathFeatures.read(insertResponse, jsonPathConf);

  final JsonPath jsonPathFeatureIds = JsonPath.compile("$.features..id");
  List<String> ids = jsonPathFeatureIds.read(insertResponse, jsonPathConf);
  logger.info("Preparation: Inserted features {}", ids);

  // =========== UPDATE ==========
  logger.info("Modify features");
  final DocumentContext updateFeaturesEventDoc = getEventFromResource("/events/InsertFeaturesEvent.json");
  updateFeaturesEventDoc.put("$", "params", Collections.singletonMap("includeOldStates", includeOldStates));

  List<Map> updateFeatures = jsonPathFeatures.read(insertResponse, jsonPathConf);
  updateFeaturesEventDoc.delete("$.insertFeatures");
  updateFeatures.forEach((Map feature) -> {
    final Map<String, Object> properties = (Map<String, Object>) feature.get("properties");
    properties.put("test", "updated");
  });
  updateFeaturesEventDoc.put("$", "updateFeatures", updateFeatures);

  String updateFeaturesEvent = updateFeaturesEventDoc.jsonString();
  String updateFeaturesResponse = invokeLambda(updateFeaturesEvent);
  assertNoErrorInResponse(updateFeaturesResponse);

  List features = jsonPathFeatures.read(updateFeaturesResponse, jsonPathConf);
  assertNotNull("'features' element in ModifyFeaturesResponse is missing", features);
  assertTrue("'features' element in ModifyFeaturesResponse is empty", features.size() > 0);

  final JsonPath jsonPathOldFeatures = JsonPath.compile("$.oldFeatures");
  List oldFeatures = jsonPathOldFeatures.read(updateFeaturesResponse, jsonPathConf);
  if (includeOldStates) {
    assertNotNull("'oldFeatures' element in ModifyFeaturesResponse is missing", oldFeatures);
    assertTrue("'oldFeatures' element in ModifyFeaturesResponse is empty", oldFeatures.size() > 0);
    assertEquals(oldFeatures, originalFeatures);
  } else if (oldFeatures != null) {
    assertEquals("unexpected oldFeatures in ModifyFeaturesResponse", 0, oldFeatures.size());
  }

  // =========== DELETE ==========
  final DocumentContext modifyFeaturesEventDoc = getEventFromResource("/events/InsertFeaturesEvent.json");
  modifyFeaturesEventDoc.put("$", "params", Collections.singletonMap("includeOldStates", includeOldStates));
  modifyFeaturesEventDoc.delete("$.insertFeatures");

  Map<String, String> idsMap = new HashMap<>();
  ids.forEach(id -> idsMap.put(id, null));
  modifyFeaturesEventDoc.put("$", "deleteFeatures", idsMap);

  String deleteEvent = modifyFeaturesEventDoc.jsonString();
  String deleteResponse = invokeLambda(deleteEvent);
  assertNoErrorInResponse(deleteResponse);
  oldFeatures = jsonPathOldFeatures.read(deleteResponse, jsonPathConf);
  if (includeOldStates) {
    assertNotNull("'oldFeatures' element in ModifyFeaturesResponse is missing", oldFeatures);
    assertTrue("'oldFeatures' element in ModifyFeaturesResponse is empty", oldFeatures.size() > 0);
    assertEquals(oldFeatures, features);
  } else if (oldFeatures != null) {
    assertEquals("unexpected oldFeatures in ModifyFeaturesResponse", 0, oldFeatures.size());
  }

  logger.info("Modify features tested successfully");
}
 

@Override
public JSONObject handle(final HttpResponse response) throws Exception {
  return new JSONObject(IOUtils.toString(response.getContent()));
}
 

@Override
public JSONArray handle(final HttpResponse response) throws Exception {
  return new JSONArray(IOUtils.toString(response.getContent()));
}
 

@Test
public void testGetStatisticsEvent() throws Exception {

  // =========== INSERT ==========
  XyzNamespace xyzNamespace = new XyzNamespace().withSpace("foo").withCreatedAt(1517504700726L);
  String insertJsonFile = "/events/InsertFeaturesEventTransactional.json";
  String insertResponse = invokeLambdaFromFile(insertJsonFile);
  String insertRequest = IOUtils.toString(GSContext.class.getResourceAsStream(insertJsonFile));
  assertRead(insertRequest, insertResponse, true);
  logger.info("Insert feature tested successfully");

  // =========== GetStatistics ==========
  GetStatisticsEvent event = new GetStatisticsEvent();
  event.setSpace("foo");
  event.setStreamId(RandomStringUtils.randomAlphanumeric(10));
  String eventJson = event.serialize();
  String statisticsJson = invokeLambda(eventJson);
  StatisticsResponse response = XyzSerializable.deserialize(statisticsJson);

  assertNotNull(response);

  assertEquals(new Long(3), response.getCount().getValue());
  assertEquals(false, response.getCount().getEstimated());

  assertTrue(response.getByteSize().getValue() > 0);
  assertEquals(true, response.getByteSize().getEstimated());

  assertNotNull(response.getBbox());
  assertEquals(false, response.getBbox().getEstimated());

  assertEquals("name", response.getProperties().getValue().get(0).getKey());
  assertEquals("string", response.getProperties().getValue().get(0).getDatatype());
  assertEquals(3, response.getProperties().getValue().get(0).getCount());
  assertEquals(false, response.getProperties().getEstimated());
  assertEquals(PropertiesStatistics.Searchable.ALL, response.getProperties().getSearchable());

  assertEquals(3, response.getTags().getValue().size());
  assertEquals(false, response.getTags().getEstimated());

  assertEquals(new ArrayList<>(Collections.singletonList("Point")), response.getGeometryTypes().getValue());
  assertEquals(false, response.getGeometryTypes().getEstimated());

  // =========== INSERT 11k ==========
  FeatureCollection collection = new FeatureCollection();
  Random random = new Random();

  List<String> pKeys = Stream.generate(() ->
      RandomStringUtils.randomAlphanumeric(10)).limit(3).collect(Collectors.toList());

  collection.setFeatures(new ArrayList<>());
  collection.getFeatures().addAll(
      Stream.generate(() -> {
        Feature f = new Feature()
            .withGeometry(
                new Point().withCoordinates(new PointCoordinates(360d * random.nextDouble() - 180d, 180d * random.nextDouble() - 90d)))
            .withProperties(new Properties().withXyzNamespace(xyzNamespace));
        pKeys.forEach(p -> f.getProperties().put(p, RandomStringUtils.randomAlphanumeric(8)));
        return f;
      }).limit(11000).collect(Collectors.toList()));

  ModifyFeaturesEvent mfevent = new ModifyFeaturesEvent();
  mfevent.setSpace("foo");
  mfevent.setTransaction(true);
  mfevent.setEnableUUID(true);
  mfevent.setInsertFeatures(collection.getFeatures());
  invokeLambda(mfevent.serialize());

  /* Needed to trigger update on pg_stat*/
  try (final Connection connection = lambda.dataSource.getConnection()) {
    Statement stmt = connection.createStatement();
    stmt.execute("ANALYZE public.\"foo\";");
  }

  statisticsJson = invokeLambda(eventJson);
  // =========== GetStatistics ==========
  response = XyzSerializable.deserialize(statisticsJson);

  assertEquals(new Long(11003), response.getCount().getValue());

  assertEquals(true, response.getCount().getEstimated());
  assertEquals(true, response.getByteSize().getEstimated());
  assertEquals(true, response.getBbox().getEstimated());
  assertEquals(true, response.getTags().getEstimated());
  assertEquals(true, response.getGeometryTypes().getEstimated());

  assertEquals(PropertiesStatistics.Searchable.PARTIAL, response.getProperties().getSearchable());

  for (PropertyStatistics prop : response.getProperties().getValue()) {
    if(prop.getKey().equalsIgnoreCase("name"))
      continue;
    assertTrue(pKeys.contains(prop.getKey()));
    assertEquals(prop.getCount() > 10000, true);
  }
}