Java Code Examples for com.amazonaws.services.s3.AmazonS3Client#getObject()

/**
 * @return the new config file (datapipelines.properties) from s3 as a String
 */
private static String getNewConfig () {
  StringBuilder config = new StringBuilder();

  // get config object from S3
  AmazonS3Client s3Client = new AmazonS3Client();
  S3Object configObj = s3Client.getObject(DATAPIPELINES_CONFIG_S3_BUCKET,
      DATAPIPELINES_CONFIG_S3_KEY);

  // write object to String
  BufferedReader reader = new BufferedReader(new InputStreamReader(configObj.getObjectContent()));
  String line;
  try {
    while ((line = reader.readLine()) != null) {
      config.append(String.format("%s\n", line));
    }
  } catch (IOException e) {
    LOG.error("Failed to read config ({}) S3Object to String, exception: {}.",
        String.format("%s/%s", DATAPIPELINES_CONFIG_S3_BUCKET, DATAPIPELINES_CONFIG_S3_KEY),
        ExceptionUtils.getFullStackTrace(e));
  }

  return config.toString();
}
 

@Override
public void setConf(AbstractConfig config) {
  this.config = (GeoIpOperationConfig) config;
  AmazonS3Client client = this.s3Factory.newInstance();

  AmazonS3URI uri = new AmazonS3URI(this.config.getGeoLiteDb());
  GetObjectRequest req = new GetObjectRequest(uri.getBucket(), uri.getKey());
  S3Object obj = client.getObject(req);

  try {
    this.databaseReader =
        new DatabaseReader.Builder(obj.getObjectContent()).withCache(new CHMCache()).build();
  } catch (IOException e) {
    throw new ConfigurationException("Unable to read " + this.config.getGeoLiteDb(), e);
  }
}
 

public static BenderConfig load(AmazonS3ClientFactory s3ClientFactory, AmazonS3URI s3Uri) {
  AmazonS3Client s3 = s3ClientFactory.newInstance();
  S3Object s3object = s3.getObject(s3Uri.getBucket(), s3Uri.getKey());

  StringWriter writer = new StringWriter();

  try {
    IOUtils.copy(s3object.getObjectContent(), writer, "UTF-8");
  } catch (IOException e) {
    throw new ConfigurationException("Unable to read file from s3", e);
  }
  BenderConfig config = load(s3Uri.getKey().toString(), writer.toString());
  config.setConfigFile(s3Uri.getURI().toString());

  return config;
}
 

/**
 * Connect to S3 directory to look for new or updated table definitions and then
 * update the map.
 */
private void updateTablesFromS3()
{
    long now = System.currentTimeMillis();

    AmazonS3Client s3client = clientManager.getS3Client();

    for (S3ObjectSummary summary : getObjectSummaries()) {
        if (!descriptors.containsKey(summary.getKey()) || summary.getLastModified().getTime() >= lastCheck) {
            // New or updated file, so we must read from AWS
            if (summary.getKey().endsWith("/")) {
                continue;
            }

            log.info("Getting : %s - %s", summary.getBucketName(), summary.getKey());
            S3Object object = s3client.getObject(new GetObjectRequest(summary.getBucketName(), summary.getKey()));

            try (BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent(), UTF_8))) {
                KinesisStreamDescription table = streamDescriptionCodec.fromJson(CharStreams.toString(reader));
                descriptors.put(summary.getKey(), table);
                log.info("Put table description into the map from %s", summary.getKey());
            }
            catch (IOException iox) {
                log.error("Problem reading input stream from object.", iox);
                throwIfUnchecked(iox);
                throw new RuntimeException(iox);
            }
        }
    }

    log.info("Completed updating table definitions from S3.");
    lastCheck = now;
}
 

@Override
public String downloadSchema(BackupRestoreContext ctx) throws Exception {
    final String nodeId = ctx.getNodeId();
    final AmazonS3Client amazonS3Client = getAmazonS3Client(ctx);
    final String key = getPrefixKey(ctx) + "/" + nodeId + "/" + StorageUtil.SCHEMA_FILE;

    S3Object object = amazonS3Client.getObject(
            new GetObjectRequest(getBucketName(ctx), key));
    InputStream objectData = object.getObjectContent();
    String schema = IOUtils.toString(objectData, "UTF-8");
    objectData.close();
    return schema;
}
 

@Override
public S3Object getObject(String bucket, String key) throws Exception
{
    RefCountedClient holder = client.get();
    AmazonS3Client amazonS3Client = holder.useClient();
    try
    {
        return amazonS3Client.getObject(bucket, key);
    }
    finally
    {
        holder.release();
    }
}
 

public void processDirectory(AmazonS3Client s3, ObjectListing listing, String bucket) throws Exception {
	for (S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
		long size = objectSummary.getSize();
		logger.info("*** Processing S3 {}, size: {}",objectSummary.getKey(),size);
		S3Object object = s3.getObject(new GetObjectRequest(bucket, objectSummary.getKey()));

		String bucketName = object.getBucketName();
		String keyName = object.getKey();

		GetObjectTaggingRequest request = new GetObjectTaggingRequest(bucketName, keyName);
		GetObjectTaggingResult result = s3.getObjectTagging(request);
		List<Tag> tags = result.getTagSet();
		String type = null;
		String name = null;

		if (tags.isEmpty()) {
			System.err.println("Error: " + keyName + " has no tags");
		} else {
			for (Tag tag : tags) {
				String key = tag.getKey();
				String value = tag.getValue();

				if (key.equals("type")) {
					type = value;
				}

				if (key.equals("name")) {
					name = value;
				}
			}

			if (name == null)
				throw new Exception("Error: " + keyName + " is missing a name tag");
			if (name.contains(" "))
				throw new Exception("Error: " + keyName + " has a name attribute with a space in it");
			if (type == null)
				throw new Exception("Error: " + keyName + " has no type tag");

			if (!name.startsWith("$"))
				name = "$" + name;

			readData(type, name, object, size);
		}
	}
}
 

/**
 * Constructs a new input stream which reads its data from the specified S3 object.
 * 
 * @param s3Client
 *        the S3 client to connect to Amazon S3.
 * @param bucket
 *        the name of the S3 bucket the object is stored in
 * @param object
 *        the name of the S3 object whose content shall be read
 * @throws IOException
 *         thrown if an error occurs while accessing the specified S3 object
 */
S3DataInputStream(final AmazonS3Client s3Client, final String bucket, final String object) throws IOException {

	S3Object s3o = null;
	try {
		s3o = s3Client.getObject(bucket, object);
	} catch (AmazonServiceException e) {
		throw new IOException(StringUtils.stringifyException(e));
	}

	this.inputStream = s3o.getObjectContent();
}