org.jclouds.blobstore.BlobStoreContext Java Examples

/**
 * Delete the configured s3 bucket.
 */
public void deleteBucket() {

    logger.debug("\n\nDelete bucket\n");

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(), new NettyPayloadModule());

    BlobStoreContext context =
        ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
            .overrides(overrides ).buildView(BlobStoreContext.class);

    BlobStore blobStore = context.getBlobStore();
    blobStore.deleteContainer(bucketName);
}
 

@Before
public void setUp() throws Exception {
    containerName = createRandomContainerName();

    nearContext = ContextBuilder
            .newBuilder("transient")
            .credentials("identity", "credential")
            .modules(ImmutableList.<Module>of(new SLF4JLoggingModule()))
            .build(BlobStoreContext.class);
    nearBlobStore = nearContext.getBlobStore();
    nearBlobStore.createContainerInLocation(null, containerName);

    farContext = ContextBuilder
            .newBuilder("transient")
            .credentials("identity", "credential")
            .modules(ImmutableList.<Module>of(new SLF4JLoggingModule()))
            .build(BlobStoreContext.class);
    farBlobStore = farContext.getBlobStore();
    farBlobStore.createContainerInLocation(null, containerName);

    executorService = Executors.newScheduledThreadPool(1);

    eventualBlobStore = EventualBlobStore.newEventualBlobStore(
            nearBlobStore, farBlobStore, executorService, DELAY,
            DELAY_UNIT, 1.0);
}
 

private static BlobStoreContext blobStoreContextFromProperties(
        Properties properties) {
    String provider = properties.getProperty(Constants.PROPERTY_PROVIDER);
    String identity = properties.getProperty(Constants.PROPERTY_IDENTITY);
    String credential = properties.getProperty(
            Constants.PROPERTY_CREDENTIAL);
    String endpoint = properties.getProperty(Constants.PROPERTY_ENDPOINT);
    if (provider == null || identity == null || credential == null) {
        System.err.println("Properties file must contain:\n" +
                Constants.PROPERTY_PROVIDER + "\n" +
                Constants.PROPERTY_IDENTITY + "\n" +
                Constants.PROPERTY_CREDENTIAL);
        System.exit(1);
    }

    ContextBuilder builder = ContextBuilder
            .newBuilder(provider)
            .credentials(identity, credential)
            .modules(ImmutableList.<Module>of(new SLF4JLoggingModule()))
            .overrides(properties);
    if (endpoint != null) {
        builder = builder.endpoint(endpoint);
    }
    return builder.build(BlobStoreContext.class);
}
 

@Before
public void setUp() throws Exception {
    ContextBuilder builder = ContextBuilder
            .newBuilder("transient")
            .credentials("identity", "credential")
            .modules(ImmutableList.<Module>of(new SLF4JLoggingModule()));
    context = builder.build(BlobStoreContext.class);
    contextRead = builder.build(BlobStoreContext.class);

    BlobStore blobStore = context.getBlobStore();
    BlobStore blobStoreRead = contextRead.getBlobStore();

    String containerName = "container-name";
    Location location = null;
    blobStore.createContainerInLocation(location, containerName);
    blobStoreRead.createContainerInLocation(location, containerName);

    awcyStrong = new AreWeConsistentYet(blobStore,
            blobStore, containerName, ITERATIONS, OBJECT_SIZE);
    awcyEventual = new AreWeConsistentYet(blobStore,
            blobStoreRead, containerName, ITERATIONS, OBJECT_SIZE);
}
 

public static void main(String[] args) throws IOException {
   if (args.length < 2) {
      throw new IllegalArgumentException("Invalid number of parameters. Syntax is: \"identity\" \"credential\"");
   }

   String identity = args[0];
   String credentials = args[1];

   // Init
   BlobStoreContext context = ContextBuilder.newBuilder("glacier")
         .credentials(identity, credentials)
         .buildView(BlobStoreContext.class);

   try {
      while (chooseOption(context));
   } finally {
      context.close();
   }
}
 

public synchronized BlobStoreContext getBlobStoreContext() {
        if (context==null) {
            if (location==null) {
                Preconditions.checkNotNull(locationSpec, "locationSpec required for remote object store when location is null");
                Preconditions.checkNotNull(mgmt, "mgmt not injected / object store not prepared");
                location = (JcloudsLocation) mgmt.getLocationRegistry().getLocationManaged(locationSpec);
            }
            
            context = BlobStoreContextFactoryImpl.INSTANCE.newBlobStoreContext(location);
     
            // TODO do we need to get location from region? can't see the jclouds API.
            // doesn't matter in some places because it's already in the endpoint
//            String region = location.getConfig(CloudLocationConfig.CLOUD_REGION_ID);
            context.getBlobStore().createContainerInLocation(null, getContainerNameFirstPart());
        }
        return context;
    }
 

public synchronized boolean hasNewFiles() {
    // see if there are any files since lastMarker.
    BlobStoreContext ctx = ContextBuilder.newBuilder(provider)
            .credentials(user, key)
            .overrides(new Properties() {{
                setProperty(LocationConstants.PROPERTY_ZONE, zone);
            }})
            .buildView(BlobStoreContext.class);
    
    BlobStore store = ctx.getBlobStore();
    ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker);
    PageSet<? extends StorageMetadata> pages = store.list(container, options);
    
    log.debug("Saw {} new files since {}", pages.size() == batchSize ? "many" : Integer.toString(pages.size()), lastMarker);
    boolean emptiness = getBlobsWithinRange(pages).isEmpty();

    if(emptiness) {
        log.warn("No file found within range {}", new Range(START_TIME, STOP_TIME));
    } else {
        log.debug("New files found within range {}", new Range(START_TIME, STOP_TIME));
    }

    return !emptiness;
}
 

private static Pair<BlobStoreLocation, BlobStore> createBlobStore(String driver,
                                                                  String region,
                                                                  String endpoint,
                                                                  Supplier<Credentials> credentials,
                                                                  int maxBlockSize) {
    Properties overrides = new Properties();
    // This property controls the number of parts being uploaded in parallel.
    overrides.setProperty("jclouds.mpu.parallel.degree", "1");
    overrides.setProperty("jclouds.mpu.parts.size", Integer.toString(maxBlockSize));
    overrides.setProperty(Constants.PROPERTY_SO_TIMEOUT, "25000");
    overrides.setProperty(Constants.PROPERTY_MAX_RETRIES, Integer.toString(100));

    ApiRegistry.registerApi(new S3ApiMetadata());
    ProviderRegistry.registerProvider(new AWSS3ProviderMetadata());
    ProviderRegistry.registerProvider(new GoogleCloudStorageProviderMetadata());

    ContextBuilder contextBuilder = ContextBuilder.newBuilder(driver);
    contextBuilder.credentialsSupplier(credentials);

    if (isS3Driver(driver) && !Strings.isNullOrEmpty(endpoint)) {
        contextBuilder.endpoint(endpoint);
        overrides.setProperty(S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "false");
    }
    contextBuilder.overrides(overrides);
    BlobStoreContext context = contextBuilder.buildView(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();

    log.info("Connect to blobstore : driver: {}, region: {}, endpoint: {}",
        driver, region, endpoint);
    return Pair.of(
        BlobStoreLocation.of(region, endpoint),
        blobStore);
}
 

public S3ProxyImpl(String endpoint, S3Config s3Config) {
    URI uri = URI.create(endpoint);

    Properties properties = new Properties();
    properties.setProperty("s3proxy.authorization", "none");
    properties.setProperty("s3proxy.endpoint", endpoint);
    properties.setProperty("jclouds.provider", "filesystem");
    properties.setProperty("jclouds.filesystem.basedir", "/tmp/s3proxy");

    ContextBuilder builder = ContextBuilder
            .newBuilder("filesystem")
            .credentials("x", "x")
            .modules(ImmutableList.<Module>of(new SLF4JLoggingModule()))
            .overrides(properties);
    BlobStoreContext context = builder.build(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();
    s3Proxy = S3Proxy.builder().awsAuthentication(AuthenticationType.AWS_V2_OR_V4, "x", "x")
                     .endpoint(uri)
                     .keyStore("", "")
                     .blobStore(blobStore)
                     .ignoreUnknownHeaders(true)
                     .build();
    client = new S3JerseyCopyPartClient(s3Config);
}
 

/**
 * Delete the configured s3 bucket.
 */
public void deleteBucket() {

    logger.debug("\n\nDelete bucket\n");

    String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
    String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet
        .of(new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(),
            new NettyPayloadModule());

    BlobStoreContext context =
        ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
            .overrides(overrides).buildView(BlobStoreContext.class);

    BlobStore blobStore = context.getBlobStore();
    blobStore.deleteContainer( bucketName );
}
 

public void deleteBucket() {

        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

        Properties overrides = new Properties();
        overrides.setProperty( "s3" + ".identity", accessId );
        overrides.setProperty( "s3" + ".credential", secretKey );

        Blob bo = null;
        BlobStore blobStore = null;
        final Iterable<? extends Module> MODULES = ImmutableSet
                .of(new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
                        new NettyPayloadModule());

        BlobStoreContext context =
                ContextBuilder.newBuilder("s3").credentials( accessId, secretKey ).modules( MODULES )
                        .overrides( overrides ).buildView( BlobStoreContext.class );

        blobStore = context.getBlobStore();
        blobStore.deleteContainer( bucketName );
    }
 

public CloudFilesPublisher() {
    Properties overrides = new Properties();
    overrides.setProperty(LocationConstants.PROPERTY_ZONE, ZONE);

    BlobStoreContext context = ContextBuilder.newBuilder(PROVIDER)
            .credentials(USERNAME, API_KEY)
            .overrides(overrides)
            .buildView(BlobStoreContext.class);
    blobStore = context.getBlobStore();
}
 

public synchronized void downloadNewFiles(File downloadDir) {
    log.info("Downloading new files since {}", lastMarker);
    
    BlobStoreContext ctx = ContextBuilder.newBuilder(provider)
            .credentials(user, key)
            .overrides(new Properties() {{
                setProperty(LocationConstants.PROPERTY_ZONE, zone);
            }})
            .buildView(BlobStoreContext.class);

    // threadsafe according to https://jclouds.apache.org/documentation/userguide/blobstore-guide/
    BlobStore store = ctx.getBlobStore();
    ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker);
    PageSet<? extends StorageMetadata> pages = store.list(container, options);

    //Gets key within the time range specified
    NavigableMap<Long, String> mapWithinRange = getBlobsWithinRange(pages);

    //Download only for keys within that range
    for(Map.Entry<Long, String> blobMeta : mapWithinRange.entrySet()) {
        log.info("Downloading file: " + blobMeta.getValue());
        downloadWorkers.submit(new BlobDownload(downloadDir, store, container, blobMeta.getValue()));
        lastMarker = blobMeta.getValue();
        synchronized (CloudFilesManager.this) {
            // this is where we resume from.
            MarkerUtils.writeLastMarker(blobMeta.getValue());
        }
    }
    log.info("Updated the last marker value as " + lastMarker);
}
 

private BlobStore getBlobStore() {
    BlobStore blobStore = ContextBuilder.newBuilder("transient")
        .credentials("id", "credential")
        .buildView(BlobStoreContext.class)
        .getBlobStore();
    blobStore.createContainerInLocation(null, CONTAINER_NAME);
    blobStore.createContainerInLocation(null, CONTAINER_NAME_WITH_DIR);
    return blobStore;
}
 

public static boolean chooseOption(BlobStoreContext context) throws IOException {
   Scanner scan = new Scanner(System.in);
   System.out.println("");
   System.out.println("Glacier examples");
   System.out.println("1. Put and retrieve blob (~4-5 hours)");
   System.out.println("2. Multipart upload (~1-5 minutes)");
   System.out.println("3. Call interruption (~0-2 minutes)");
   System.out.println("4. Provider API (~4-5 hours)");
   System.out.println("5. Exit");
   System.out.print("Choose an option: ");
   try{
      switch(scan.nextInt()){
      case 1:
         putAndRetrieveBlobExample(context.getBlobStore());
         break;
      case 2:
         multipartUploadExample(context.getBlobStore());
         break;
      case 3:
         interruptionExample(context.getBlobStore());
         break;
      case 4:
         providerExample(context);
         break;
      case 5:
         return false;
      default:
         System.out.println("Not a valid option");
         break;
      }
   }
   catch(InputMismatchException e) {
      System.out.println("Not a valid option");
   }
   return true;
}
 

private static void providerExample(BlobStoreContext context) throws IOException {
   // Get the provider API
   GlacierClient client = context.unwrapApi(GlacierClient.class);

   // Create a vault
   final String vaultName =  "jclouds_providerExample_" + UUID.randomUUID().toString();
   client.createVault(vaultName);

   // Upload an archive
   Payload payload = new ByteSourcePayload(buildData(16));
   payload.getContentMetadata().setContentType(PLAIN_TEXT_UTF_8.toString());
   payload.getContentMetadata().setContentLength(16L);
   String archiveId = client.uploadArchive(vaultName, payload);

   // Create an archive retrieval job request
   JobRequest archiveRetrievalJobRequest = ArchiveRetrievalJobRequest.builder()
         .archiveId(archiveId)
         .description("retrieval job")
         .build();

   // Initiate job
   String jobId = client.initiateJob(vaultName, archiveRetrievalJobRequest);
   try {
      // Poll until the job is done
      new BasePollingStrategy(client).waitForSuccess(vaultName, jobId);

      // Get the job output
      Payload result = client.getJobOutput(vaultName, jobId);

      // Print the result
      System.out.println("The retrieved payload is: " + Strings2.toStringAndClose(result.openStream()));
   } catch (InterruptedException e) {
      Throwables.propagate(e);
   }
}
 

private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix );

        String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
        String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for ( Object o : blobStoreList.toArray() ) {
            StorageMetadata s = (StorageMetadata)o;

            if ( s.getName().startsWith( bucketPrefix )) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch ( ContainerNotFoundException cnfe ) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe );
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
 

/**
 * Initializes the BlobStore context.
 */
public void init() {
    Iterable<Module> modules = ImmutableSet.<Module>of(new SLF4JLoggingModule());

    // The metadata must be registered because the ServiceLoader does not detect
    // the META-INF/services in each api JAR under Tomcat/Spring. Fortunately,
    // the registry is static and is not really OSGi specific for this use.
    ApiRegistry.registerApi(new SwiftApiMetadata());
    ProviderRegistry.registerProvider(new AWSS3ProviderMetadata());

    context = ContextBuilder.newBuilder(provider)
            .credentials(identity, credential)
            .modules(modules)
            .buildView(BlobStoreContext.class);

    // There are some oddities with streaming larger files to the user,
    // so download to a temp file first. For now, call 100MB the threshold.
    maxBlobStreamSize = (long) serverConfigurationService.getInt("cloud.content.maxblobstream.size", 1024 * 1024 * 100);
    temporaryBlobDirectory = serverConfigurationService.getString("cloud.content.temporary.directory", null);
    
    if (temporaryBlobDirectory != null) {
        File baseDir = new File(temporaryBlobDirectory);
        if (!baseDir.exists()) {
            try {
                // Can't write into the preferred temp dir
                if (!baseDir.mkdirs()) {
                    temporaryBlobDirectory = null;
                }
            }
            catch (SecurityException se) {
                // JVM security hasn't whitelisted this dir
                temporaryBlobDirectory = null;
            }
        }
    }
}
 

@Before
public void setUp() throws Exception {
    containerName = createRandomContainerName();

    context = ContextBuilder
            .newBuilder("transient")
            .credentials("identity", "credential")
            .modules(ImmutableList.<Module>of(new SLF4JLoggingModule()))
            .build(BlobStoreContext.class);
    blobStore = context.getBlobStore();
    blobStore.createContainerInLocation(null, containerName);
    readOnlyBlobStore = ReadOnlyBlobStore.newReadOnlyBlobStore(blobStore);
}
 

private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix);

        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides ).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for (Object o : blobStoreList.toArray()) {
            StorageMetadata s = (StorageMetadata) o;

            if (s.getName().startsWith(bucketPrefix)) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch (ContainerNotFoundException cnfe) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe);
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
 

@Before
public void setUp() throws Exception {
    containerName = createRandomContainerName();

    context = ContextBuilder
            .newBuilder("transient")
            .credentials("identity", "credential")
            .modules(ImmutableList.<Module>of(new SLF4JLoggingModule()))
            .build(BlobStoreContext.class);
    blobStore = context.getBlobStore();
    blobStore.createContainerInLocation(null, containerName);

    nullBlobStore = NullBlobStore.newNullBlobStore(blobStore);
}
 

@BeforeMethod
public void start() throws Exception {
    if (Boolean.parseBoolean(System.getProperty("testRealAWS", "false"))) {
        log.info("TestReal AWS S3, bucket: {}", BUCKET);
        // To use this, must config credentials using "aws_access_key_id" as S3ID,
        // and "aws_secret_access_key" as S3Key. And bucket should exist in default region. e.g.
        //        props.setProperty("S3ID", "AXXXXXXQ");
        //        props.setProperty("S3Key", "HXXXXXß");
        context = ContextBuilder.newBuilder("aws-s3")
            .credentials(System.getProperty("S3ID"), System.getProperty("S3Key"))
            .build(BlobStoreContext.class);
        blobStore = context.getBlobStore();
        // To use this, ~/.aws must be configured with credentials and a default region
        //s3client = AmazonS3ClientBuilder.standard().build();
    } else if (Boolean.parseBoolean(System.getProperty("testRealGCS", "false"))) {
        log.info("TestReal GCS, bucket: {}", BUCKET);
        // To use this, must config credentials using "client_email" as GCSID and "private_key" as GCSKey.
        // And bucket should exist in default region. e.g.
        //        props.setProperty("GCSID", "[email protected]");
        //        props.setProperty("GCSKey", "XXXXXX");
        context = ContextBuilder.newBuilder("google-cloud-storage")
            .credentials(System.getProperty("GCSID"), System.getProperty("GCSKey"))
            .build(BlobStoreContext.class);
        blobStore = context.getBlobStore();
    } else {
        context = ContextBuilder.newBuilder("transient").build(BlobStoreContext.class);
        blobStore = context.getBlobStore();
        boolean create = blobStore.createContainerInLocation(null, BUCKET);
        log.debug("TestBase Create Bucket: {}, in blobStore, result: {}", BUCKET, create);
    }
}
 

@Before
public void init() {
    AliOSSApi aliOSSApi = Mockito.mock(AliOSSApi.class);
    ossClient = Mockito.spy(OSS.class);
    Mockito.when(aliOSSApi.getOSSClient()).thenReturn(ossClient);

    context = Mockito.mock(BlobStoreContext.class);
    blobUtils = Mockito.mock(BlobUtils.class);
    PayloadSlicer slicer = new BasePayloadSlicer();
    LocationsSupplier locations = () -> Sets.newHashSet(LOCATION);
    Supplier<Location> defaultLocation = () -> LOCATION;
    aliOSSBlobStore = new AliOSSBlobStore(aliOSSApi, context, blobUtils, defaultLocation, locations, slicer);
}
 

/**
 * Initializes the BlobStore context.
 */
public void init() {
    Iterable<Module> modules = ImmutableSet.<Module>of(new SLF4JLoggingModule());

    // The metadata must be registered because the ServiceLoader does not detect
    // the META-INF/services in each api JAR under Tomcat/Spring. Fortunately,
    // the registry is static and is not really OSGi specific for this use.
    ApiRegistry.registerApi(new SwiftApiMetadata());
    ProviderRegistry.registerProvider(new AWSS3ProviderMetadata());

    context = ContextBuilder.newBuilder(provider)
            .credentials(identity, credential)
            .modules(modules)
            .buildView(BlobStoreContext.class);

    // There are some oddities with streaming larger files to the user,
    // so download to a temp file first. For now, call 100MB the threshold.
    maxBlobStreamSize = (long) serverConfigurationService.getInt("cloud.content.maxblobstream.size", 1024 * 1024 * 100);
    temporaryBlobDirectory = serverConfigurationService.getString("cloud.content.temporary.directory", null);
    
    if (temporaryBlobDirectory != null) {
        File baseDir = new File(temporaryBlobDirectory);
        if (!baseDir.exists()) {
            try {
                // Can't write into the preferred temp dir
                if (!baseDir.mkdirs()) {
                    temporaryBlobDirectory = null;
                }
            }
            catch (SecurityException se) {
                // JVM security hasn't whitelisted this dir
                temporaryBlobDirectory = null;
            }
        }
    }
}
 

public BlobStore get() {
    Properties overrides = new Properties();
    overrides.setProperty("PROPERTY_S3_VIRTUAL_HOST_BUCKETS", "false");

    return contextBuilder()
        .endpoint(configuration.getEndpoint())
        .credentials(configuration.getAccessKeyId(), configuration.getSecretKey())
        .overrides(overrides)
        .modules(JCLOUDS_MODULES)
        .buildView(BlobStoreContext.class)
        .getBlobStore();
}
 

@Override
public BlobStoreContext newBlobStoreContext(Location location) {
    String rawProvider = checkNotNull(location.getConfig(LocationConfigKeys.CLOUD_PROVIDER), "provider must not be null");
    String provider = DeserializingJcloudsRenamesProvider.INSTANCE.applyJcloudsRenames(rawProvider);
    String identity = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_IDENTITY), "identity must not be null");
    String credential = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_CREDENTIAL), "credential must not be null");
    String endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);

    Properties overrides = new Properties();
    // * Java 7,8 bug workaround - sockets closed by GC break the internal bookkeeping
    //   of HttpUrlConnection, leading to invalid handling of the "HTTP/1.1 100 Continue"
    //   response. Coupled with a bug when using SSL sockets reads will block
    //   indefinitely even though a read timeout is explicitly set.
    // * Java 6 ignores the header anyways as it is included in its restricted headers black list.
    // * Also there's a bug in SL object store which still expects Content-Length bytes
    //   even when it responds with a 408 timeout response, leading to incorrectly
    //   interpreting the next request (triggered by above problem).
    overrides.setProperty(Constants.PROPERTY_STRIP_EXPECT_HEADER, "true");

    // Add extra jclouds-specific configuration
    Map<String, Object> extra = Maps.filterKeys(((LocationInternal)location).config().getBag().getAllConfig(), Predicates.containsPattern("^jclouds\\."));
    if (extra.size() > 0) {
        LOG.debug("Configuring custom jclouds property overrides for {}: {}", provider, Sanitizer.sanitize(extra));
    }
    overrides.putAll(Maps.filterValues(extra, Predicates.notNull()));

    ContextBuilder contextBuilder = ContextBuilder.newBuilder(provider).credentials(identity, credential);
    contextBuilder.modules(MutableList.copyOf(getCommonModules()));
    if (!org.apache.brooklyn.util.text.Strings.isBlank(endpoint)) {
        contextBuilder.endpoint(endpoint);
    }
    contextBuilder.overrides(overrides);
    BlobStoreContext context = contextBuilder.buildView(BlobStoreContext.class);
    return context;
}
 

private Set<Service> getServices(Credentials creds) throws Exception {
    BlobStoreContext tmpContext = BlobStoreContextFactoryImpl.INSTANCE.newBlobStoreContext(location);
    try {
        tmpContext.getBlobStore().list();
        LoadingCache<Credentials, Access> authCache = getAuthCache(tmpContext);
        Access tmpAccess = authCache.get(creds);
        return ImmutableSet.copyOf(tmpAccess);
    } finally {
        tmpContext.close();
    }

}
 

public static void main(String[] args) {
        LocalManagementContextForTests mgmt = new LocalManagementContextForTests(BrooklynProperties.Factory.newDefault()); 
        JcloudsLocation location = (JcloudsLocation) mgmt.getLocationRegistry().getLocationManaged(locationSpec);
            
        String identity = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_IDENTITY), "identity must not be null");
        String credential = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_CREDENTIAL), "credential must not be null");
        String provider = checkNotNull(location.getConfig(LocationConfigKeys.CLOUD_PROVIDER), "provider must not be null");
        String endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);

        BlobStoreContext context = ContextBuilder.newBuilder(provider)
                .credentials(identity, credential)
                .endpoint(endpoint)
                .buildView(BlobStoreContext.class);
        
        PageSet<? extends StorageMetadata> containers = context.getBlobStore().list();
        for (StorageMetadata container: containers) {
            if (container.getName().matches("brooklyn.*-test.*")
                // to kill all containers here
//                || container.getName().matches(".*")
                ) {
                log.info("killing - "+container.getName());
                context.getBlobStore().deleteContainer(container.getName());
            } else {
                log.info("KEEPING - "+container.getName());
            }
        }
        context.close();
    }
 

@Override
protected void doCall(JcloudsLocation loc, String indent) throws Exception {
    BlobStoreContext context = BlobStoreContextFactoryImpl.INSTANCE.newBlobStoreContext(loc);
    try {
        org.jclouds.blobstore.BlobStore blobStore = context.getBlobStore();
        doCall(blobStore, indent);
    } finally {
        context.close();
    }
}
 

private S3ProxyRule(Builder builder) {
    accessKey = builder.accessKey;
    secretKey = builder.secretKey;

    Properties properties = new Properties();
    try {
        blobStoreLocation = Files.createTempDirectory("S3ProxyRule")
                .toFile();
        properties.setProperty("jclouds.filesystem.basedir",
            blobStoreLocation.getCanonicalPath());
    } catch (IOException e) {
        throw new RuntimeException("Unable to initialize Blob Store", e);
    }

    blobStoreContext = ContextBuilder.newBuilder(
                builder.blobStoreProvider)
            .credentials(accessKey, secretKey)
            .overrides(properties).build(BlobStoreContext.class);

    S3Proxy.Builder s3ProxyBuilder = S3Proxy.builder()
        .blobStore(blobStoreContext.getBlobStore())
        .awsAuthentication(builder.authType, accessKey, secretKey)
        .ignoreUnknownHeaders(builder.ignoreUnknownHeaders);

    if (builder.secretStorePath != null ||
            builder.secretStorePassword != null) {
        s3ProxyBuilder.keyStore(builder.secretStorePath,
            builder.secretStorePassword);
    }

    int port = builder.port < 0 ? 0 : builder.port;
    endpointFormat = "http://%s:%d";
    String endpoint = String.format(endpointFormat, LOCALHOST, port);
    s3ProxyBuilder.endpoint(URI.create(endpoint));

    s3Proxy = s3ProxyBuilder.build();
}