Java Code Examples for com.amazonaws.services.s3.AmazonS3Client#setEndpoint()

The following examples show how to use com.amazonaws.services.s3.AmazonS3Client#setEndpoint() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: S3StorageDriver.java    From dcos-cassandra-service with Apache License 2.0 6 votes vote down vote up 
private AmazonS3Client getAmazonS3Client(BackupRestoreContext ctx) throws URISyntaxException {
    final String accessKey = ctx.getAccountId();
    final String secretKey = ctx.getSecretKey();
    String endpoint = getEndpoint(ctx);
    LOGGER.info("endpoint: {}", endpoint);

    final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(accessKey, secretKey);
    final AmazonS3Client amazonS3Client = new AmazonS3Client(basicAWSCredentials);
    amazonS3Client.setEndpoint(endpoint);

    if (ctx.usesEmc()) {
        final S3ClientOptions options = new S3ClientOptions();
        options.setPathStyleAccess(true);
        amazonS3Client.setS3ClientOptions(options);
    }

    return amazonS3Client;
}
Example 2
Source File: S3StorageIT.java    From digdag with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp()
        throws Exception
{
    assumeThat(TEST_S3_ENDPOINT, not(isEmptyOrNullString()));

    projectDir = folder.getRoot().toPath().resolve("foobar");
    config = folder.newFile().toPath();

    client = DigdagClient.builder()
            .host(server.host())
            .port(server.port())
            .build();

    AWSCredentials credentials = new BasicAWSCredentials(TEST_S3_ACCESS_KEY_ID, TEST_S3_SECRET_ACCESS_KEY);
    s3 = new AmazonS3Client(credentials);
    s3.setEndpoint(TEST_S3_ENDPOINT);

    s3.createBucket(archiveBucket);
    s3.createBucket(logStorageBucket);
}
Example 3
Source File: S3StorageFactory.java    From digdag with Apache License 2.0 6 votes vote down vote up 
@Override
public Storage newStorage(Config config)
{
    AmazonS3Client client = new AmazonS3Client(
            buildCredentialsProvider(config),
            buildClientConfiguration(config));
    if (config.has("endpoint")) {
        client.setEndpoint(config.get("endpoint", String.class));
    }

    if (config.has("path-style-access")) {
        client.setS3ClientOptions(
          S3ClientOptions.builder().setPathStyleAccess(
            config.get("path-style-access", Boolean.class, false)
          ).build());
    }

    String bucket = config.get("bucket", String.class);

    return new S3Storage(config, client, bucket);
}
Example 4
Source File: S3StorageTest.java    From digdag with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp()
        throws Exception
{
    assumeThat(TEST_S3_ENDPOINT, not(isEmptyOrNullString()));

    AWSCredentials credentials = new BasicAWSCredentials(TEST_S3_ACCESS_KEY_ID, TEST_S3_SECRET_ACCESS_KEY);
    AmazonS3Client s3 = new AmazonS3Client(credentials);
    s3.setEndpoint(TEST_S3_ENDPOINT);

    String bucket = UUID.randomUUID().toString();
    s3.createBucket(bucket);

    ConfigFactory cf = new ConfigFactory(objectMapper());
    Config config = cf.create()
        .set("endpoint", TEST_S3_ENDPOINT)
        .set("bucket", bucket)  // use unique bucket name
        .set("credentials.access-key-id", TEST_S3_ACCESS_KEY_ID)
        .set("credentials.secret-access-key", TEST_S3_SECRET_ACCESS_KEY)
        ;
    storage = new S3StorageFactory().newStorage(config);
}
Example 5
Source File: HadoopFileUtils.java    From mrgeo with Apache License 2.0 6 votes vote down vote up 
/**
 * Return an AmazonS3Client set up with the proper endpoint
 * defined in core-site.xml using a property like fs.s3a.endpoint.
 * This mimics code found in S3AFileSystem.
 *
 * @param conf
 * @param scheme
 * @return
 */
private static AmazonS3Client getS3Client(Configuration conf, String scheme)
{
  AmazonS3Client s3Client = new AmazonS3Client(new DefaultAWSCredentialsProviderChain());
  String endpointKey = "fs." + scheme.toLowerCase() + ".endpoint";
  String endPoint = conf.getTrimmed(endpointKey,"");
  log.debug("Using endpoint setting " + endpointKey);
  if (!endPoint.isEmpty()) {
    try {
      log.debug("Setting S3 client endpoint to " + endPoint);
      s3Client.setEndpoint(endPoint);
    } catch (IllegalArgumentException e) {
      String msg = "Incorrect endpoint: "  + e.getMessage();
      log.error(msg);
      throw new IllegalArgumentException(msg, e);
    }
  }
  return s3Client;
}
Example 6
Source File: AbstractS3CacheBolt.java    From storm-crawler with Apache License 2.0 6 votes vote down vote up 
/** Returns an S3 client given the configuration **/
public static AmazonS3Client getS3Client(Map conf) {
    AWSCredentialsProvider provider = new DefaultAWSCredentialsProviderChain();
    AWSCredentials credentials = provider.getCredentials();
    ClientConfiguration config = new ClientConfiguration();

    AmazonS3Client client = new AmazonS3Client(credentials, config);

    String regionName = ConfUtils.getString(conf, REGION);
    if (StringUtils.isNotBlank(regionName)) {
        client.setRegion(RegionUtils.getRegion(regionName));
    }

    String endpoint = ConfUtils.getString(conf, ENDPOINT);
    if (StringUtils.isNotBlank(endpoint)) {
        client.setEndpoint(endpoint);
    }
    return client;
}
Example 7
Source File: RedshiftManifestEmitter.java    From amazon-kinesis-connectors with Apache License 2.0 6 votes vote down vote up 
public RedshiftManifestEmitter(KinesisConnectorConfiguration configuration) {
    dataTable = configuration.REDSHIFT_DATA_TABLE;
    fileTable = configuration.REDSHIFT_FILE_TABLE;
    fileKeyColumn = configuration.REDSHIFT_FILE_KEY_COLUMN;
    dataDelimiter = configuration.REDSHIFT_DATA_DELIMITER;
    copyMandatory = configuration.REDSHIFT_COPY_MANDATORY;
    s3Bucket = configuration.S3_BUCKET;
    s3Endpoint = configuration.S3_ENDPOINT;
    s3Client = new AmazonS3Client(configuration.AWS_CREDENTIALS_PROVIDER);
    if (s3Endpoint != null) {
        s3Client.setEndpoint(s3Endpoint);
    }
    credentialsProvider = configuration.AWS_CREDENTIALS_PROVIDER;
    loginProps = new Properties();
    loginProps.setProperty("user", configuration.REDSHIFT_USERNAME);
    loginProps.setProperty("password", configuration.REDSHIFT_PASSWORD);
    redshiftURL = configuration.REDSHIFT_URL;
}
Example 8
Source File: S3WaitIT.java    From digdag with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp()
        throws Exception
{
    assumeThat(TEST_S3_ENDPOINT, not(isEmptyOrNullString()));

    proxyServer = TestUtils.startRequestFailingProxy(10);

    server = TemporaryDigdagServer.builder()
            .environment(ImmutableMap.of(
                    "http_proxy", "http://" + proxyServer.getListenAddress().getHostString() + ":" + proxyServer.getListenAddress().getPort())
            )
            .configuration(
                    "digdag.secret-encryption-key = " + Base64.getEncoder().encodeToString(RandomUtils.nextBytes(16)))
            .build();

    server.start();

    projectDir = folder.getRoot().toPath().resolve("foobar");

    client = DigdagClient.builder()
            .host(server.host())
            .port(server.port())
            .build();

    bucket = UUID.randomUUID().toString();

    AWSCredentials credentials = new BasicAWSCredentials(TEST_S3_ACCESS_KEY_ID, TEST_S3_SECRET_ACCESS_KEY);
    s3 = new AmazonS3Client(credentials);
    s3.setEndpoint(TEST_S3_ENDPOINT);
    s3.createBucket(bucket);
}
Example 9
Source File: S3RecordReader.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up 
/**
 * Initialize the AmazonS3 client using the accessKey, secretAccessKey, sets
 * endpoint for the s3Client if provided
 *
 * @param accessKey
 * @param secretAccessKey
 * @param endPoint
 */
public void initializeS3Client(@javax.validation.constraints.NotNull String accessKey,
    @javax.validation.constraints.NotNull String secretAccessKey, String endPoint)
{
  Preconditions.checkNotNull(accessKey);
  Preconditions.checkNotNull(secretAccessKey);
  s3Client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretAccessKey));
  if (endPoint != null) {
    s3Client.setEndpoint(endPoint);
  }
}
Example 10
Source File: TcpDiscoveryS3IpFinder.java    From ignite with Apache License 2.0 5 votes vote down vote up 
/**
 * Instantiates {@code AmazonS3Client} instance.
 *
 * @return Client instance to use to connect to AWS.
 */
AmazonS3Client createAmazonS3Client() {
    AmazonS3Client cln = cfg != null
        ? (cred != null ? new AmazonS3Client(cred, cfg) : new AmazonS3Client(credProvider, cfg))
        : (cred != null ? new AmazonS3Client(cred) : new AmazonS3Client(credProvider));

    if (!F.isEmpty(bucketEndpoint))
        cln.setEndpoint(bucketEndpoint);

    return cln;
}
Example 11
Source File: S3ClientFactory.java    From front50 with Apache License 2.0 5 votes vote down vote up 
public static AmazonS3 create(
    AWSCredentialsProvider awsCredentialsProvider, S3Properties s3Properties) {
  ClientConfiguration clientConfiguration = new ClientConfiguration();
  if (s3Properties.getProxyProtocol() != null) {
    if (s3Properties.getProxyProtocol().equalsIgnoreCase("HTTPS")) {
      clientConfiguration.setProtocol(Protocol.HTTPS);
    } else {
      clientConfiguration.setProtocol(Protocol.HTTP);
    }
    Optional.ofNullable(s3Properties.getProxyHost()).ifPresent(clientConfiguration::setProxyHost);
    Optional.ofNullable(s3Properties.getProxyPort())
        .map(Integer::parseInt)
        .ifPresent(clientConfiguration::setProxyPort);
  }

  AmazonS3Client client = new AmazonS3Client(awsCredentialsProvider, clientConfiguration);

  if (!StringUtils.isEmpty(s3Properties.getEndpoint())) {
    client.setEndpoint(s3Properties.getEndpoint());

    if (!StringUtils.isEmpty(s3Properties.getRegionOverride())) {
      client.setSignerRegionOverride(s3Properties.getRegionOverride());
    }

    client.setS3ClientOptions(
        S3ClientOptions.builder().setPathStyleAccess(s3Properties.getPathStyleAccess()).build());
  } else {
    Optional.ofNullable(s3Properties.getRegion())
        .map(Regions::fromName)
        .map(Region::getRegion)
        .ifPresent(client::setRegion);
  }

  return client;
}
Example 12
Source File: S3Emitter.java    From amazon-kinesis-connectors with Apache License 2.0 5 votes vote down vote up 
public S3Emitter(KinesisConnectorConfiguration configuration) {
    s3Bucket = configuration.S3_BUCKET;
    s3Endpoint = configuration.S3_ENDPOINT;
    s3client = new AmazonS3Client(configuration.AWS_CREDENTIALS_PROVIDER);
    if (s3Endpoint != null) {
        s3client.setEndpoint(s3Endpoint);
    }
}
Example 13
Source File: KinesisConnectorExecutor.java    From amazon-kinesis-connectors with Apache License 2.0 5 votes vote down vote up 
/**
 * Helper method to create the Amazon S3 bucket.
 * 
 * @param s3Bucket
 *        The name of the bucket to create
 */
private void createS3Bucket(String s3Bucket) {
    AmazonS3Client client = new AmazonS3Client(config.AWS_CREDENTIALS_PROVIDER);
    client.setEndpoint(config.S3_ENDPOINT);
    LOG.info("Creating Amazon S3 bucket " + s3Bucket);
    S3Utils.createBucket(client, s3Bucket);
}
Example 14
Source File: AWSClientFactory.java    From aws-codebuild-jenkins-plugin with Apache License 2.0 4 votes vote down vote up 
public AmazonS3Client getS3Client() throws InvalidInputException {
    AmazonS3Client client = new AmazonS3Client(awsCredentialsProvider, getClientConfiguration());
    client.setEndpoint("https://s3." + region + getAwsClientSuffix(region));
    return client;
}
Example 15
Source File: BeanstalkDeployer.java    From gradle-beanstalk-plugin with MIT License 4 votes vote down vote up 
public BeanstalkDeployer(String s3Endpoint, String beanstalkEndpoint, AWSCredentialsProvider credentialsProvider) {
    s3 = new AmazonS3Client(credentialsProvider);
    elasticBeanstalk = new AWSElasticBeanstalkClient(credentialsProvider);
    s3.setEndpoint(s3Endpoint);
    elasticBeanstalk.setEndpoint(beanstalkEndpoint);
}
Example 16
Source File: S3Utils.java    From cloudstack with Apache License 2.0 4 votes vote down vote up 
public static TransferManager getTransferManager(final ClientOptions clientOptions) {

        if(TRANSFERMANAGER_ACCESSKEY_MAP.containsKey(clientOptions.getAccessKey())) {
            return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
        }

        final AWSCredentials basicAWSCredentials = new BasicAWSCredentials(clientOptions.getAccessKey(), clientOptions.getSecretKey());

        final ClientConfiguration configuration = new ClientConfiguration();

        if (clientOptions.isHttps() != null) {
            configuration.setProtocol(clientOptions.isHttps() ? HTTPS : HTTP);
        }

        if (clientOptions.getConnectionTimeout() != null) {
            configuration.setConnectionTimeout(clientOptions.getConnectionTimeout());
        }

        if (clientOptions.getMaxErrorRetry() != null) {
            configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
        }

        if (clientOptions.getSocketTimeout() != null) {
            configuration.setSocketTimeout(clientOptions.getSocketTimeout());
        }

        if (clientOptions.getUseTCPKeepAlive() != null) {
            configuration.setUseTcpKeepAlive(clientOptions.getUseTCPKeepAlive());
        }

        if (clientOptions.getConnectionTtl() != null) {
            configuration.setConnectionTTL(clientOptions.getConnectionTtl());
        }

        if (clientOptions.getSigner() != null) {

            configuration.setSignerOverride(clientOptions.getSigner());
        }

        LOGGER.debug(format("Creating S3 client with configuration: [protocol: %1$s, signer: %2$s, connectionTimeOut: %3$s, maxErrorRetry: %4$s, socketTimeout: %5$s, useTCPKeepAlive: %6$s, connectionTtl: %7$s]",
                configuration.getProtocol(), configuration.getSignerOverride(), configuration.getConnectionTimeout(), configuration.getMaxErrorRetry(), configuration.getSocketTimeout(),
                clientOptions.getUseTCPKeepAlive(), clientOptions.getConnectionTtl()));

        final AmazonS3Client client = new AmazonS3Client(basicAWSCredentials, configuration);

        if (isNotBlank(clientOptions.getEndPoint())) {
            LOGGER.debug(format("Setting the end point for S3 client with access key %1$s to %2$s.", clientOptions.getAccessKey(), clientOptions.getEndPoint()));

            client.setEndpoint(clientOptions.getEndPoint());
        }

        TRANSFERMANAGER_ACCESSKEY_MAP.put(clientOptions.getAccessKey(), new TransferManager(client));

        return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
    }