Java Code Examples for org.jclouds.blobstore.BlobStoreContext#getBlobStore()

public S3ProxyImpl(String endpoint, S3Config s3Config) {
    URI uri = URI.create(endpoint);

    Properties properties = new Properties();
    properties.setProperty("s3proxy.authorization", "none");
    properties.setProperty("s3proxy.endpoint", endpoint);
    properties.setProperty("jclouds.provider", "filesystem");
    properties.setProperty("jclouds.filesystem.basedir", "/tmp/s3proxy");

    ContextBuilder builder = ContextBuilder
            .newBuilder("filesystem")
            .credentials("x", "x")
            .modules(ImmutableList.<Module>of(new SLF4JLoggingModule()))
            .overrides(properties);
    BlobStoreContext context = builder.build(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();
    s3Proxy = S3Proxy.builder().awsAuthentication(AuthenticationType.AWS_V2_OR_V4, "x", "x")
                     .endpoint(uri)
                     .keyStore("", "")
                     .blobStore(blobStore)
                     .ignoreUnknownHeaders(true)
                     .build();
    client = new S3JerseyCopyPartClient(s3Config);
}
 

private static Pair<BlobStoreLocation, BlobStore> createBlobStore(String driver,
                                                                  String region,
                                                                  String endpoint,
                                                                  Supplier<Credentials> credentials,
                                                                  int maxBlockSize) {
    Properties overrides = new Properties();
    // This property controls the number of parts being uploaded in parallel.
    overrides.setProperty("jclouds.mpu.parallel.degree", "1");
    overrides.setProperty("jclouds.mpu.parts.size", Integer.toString(maxBlockSize));
    overrides.setProperty(Constants.PROPERTY_SO_TIMEOUT, "25000");
    overrides.setProperty(Constants.PROPERTY_MAX_RETRIES, Integer.toString(100));

    ApiRegistry.registerApi(new S3ApiMetadata());
    ProviderRegistry.registerProvider(new AWSS3ProviderMetadata());
    ProviderRegistry.registerProvider(new GoogleCloudStorageProviderMetadata());

    ContextBuilder contextBuilder = ContextBuilder.newBuilder(driver);
    contextBuilder.credentialsSupplier(credentials);

    if (isS3Driver(driver) && !Strings.isNullOrEmpty(endpoint)) {
        contextBuilder.endpoint(endpoint);
        overrides.setProperty(S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "false");
    }
    contextBuilder.overrides(overrides);
    BlobStoreContext context = contextBuilder.buildView(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();

    log.info("Connect to blobstore : driver: {}, region: {}, endpoint: {}",
        driver, region, endpoint);
    return Pair.of(
        BlobStoreLocation.of(region, endpoint),
        blobStore);
}
 

/**
 * Delete the configured s3 bucket.
 */
public void deleteBucket() {

    logger.debug("\n\nDelete bucket\n");

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(), new NettyPayloadModule());

    BlobStoreContext context =
        ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
            .overrides(overrides ).buildView(BlobStoreContext.class);

    BlobStore blobStore = context.getBlobStore();
    blobStore.deleteContainer(bucketName);
}
 

public void deleteBucket() {

        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

        Properties overrides = new Properties();
        overrides.setProperty( "s3" + ".identity", accessId );
        overrides.setProperty( "s3" + ".credential", secretKey );

        Blob bo = null;
        BlobStore blobStore = null;
        final Iterable<? extends Module> MODULES = ImmutableSet
                .of(new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
                        new NettyPayloadModule());

        BlobStoreContext context =
                ContextBuilder.newBuilder("s3").credentials( accessId, secretKey ).modules( MODULES )
                        .overrides( overrides ).buildView( BlobStoreContext.class );

        blobStore = context.getBlobStore();
        blobStore.deleteContainer( bucketName );
    }
 

public synchronized boolean hasNewFiles() {
    // see if there are any files since lastMarker.
    BlobStoreContext ctx = ContextBuilder.newBuilder(provider)
            .credentials(user, key)
            .overrides(new Properties() {{
                setProperty(LocationConstants.PROPERTY_ZONE, zone);
            }})
            .buildView(BlobStoreContext.class);
    
    BlobStore store = ctx.getBlobStore();
    ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker);
    PageSet<? extends StorageMetadata> pages = store.list(container, options);
    
    log.debug("Saw {} new files since {}", pages.size() == batchSize ? "many" : Integer.toString(pages.size()), lastMarker);
    boolean emptiness = getBlobsWithinRange(pages).isEmpty();

    if(emptiness) {
        log.warn("No file found within range {}", new Range(START_TIME, STOP_TIME));
    } else {
        log.debug("New files found within range {}", new Range(START_TIME, STOP_TIME));
    }

    return !emptiness;
}
 

/**
 * Delete the configured s3 bucket.
 */
public void deleteBucket() {

    logger.debug("\n\nDelete bucket\n");

    String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
    String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet
        .of(new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(),
            new NettyPayloadModule());

    BlobStoreContext context =
        ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
            .overrides(overrides).buildView(BlobStoreContext.class);

    BlobStore blobStore = context.getBlobStore();
    blobStore.deleteContainer( bucketName );
}
 

private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix);

        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides ).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for (Object o : blobStoreList.toArray()) {
            StorageMetadata s = (StorageMetadata) o;

            if (s.getName().startsWith(bucketPrefix)) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch (ContainerNotFoundException cnfe) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe);
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
 

private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix );

        String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
        String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for ( Object o : blobStoreList.toArray() ) {
            StorageMetadata s = (StorageMetadata)o;

            if ( s.getName().startsWith( bucketPrefix )) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch ( ContainerNotFoundException cnfe ) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe );
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
 

public CloudFilesPublisher() {
    Properties overrides = new Properties();
    overrides.setProperty(LocationConstants.PROPERTY_ZONE, ZONE);

    BlobStoreContext context = ContextBuilder.newBuilder(PROVIDER)
            .credentials(USERNAME, API_KEY)
            .overrides(overrides)
            .buildView(BlobStoreContext.class);
    blobStore = context.getBlobStore();
}
 

public synchronized void downloadNewFiles(File downloadDir) {
    log.info("Downloading new files since {}", lastMarker);
    
    BlobStoreContext ctx = ContextBuilder.newBuilder(provider)
            .credentials(user, key)
            .overrides(new Properties() {{
                setProperty(LocationConstants.PROPERTY_ZONE, zone);
            }})
            .buildView(BlobStoreContext.class);

    // threadsafe according to https://jclouds.apache.org/documentation/userguide/blobstore-guide/
    BlobStore store = ctx.getBlobStore();
    ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker);
    PageSet<? extends StorageMetadata> pages = store.list(container, options);

    //Gets key within the time range specified
    NavigableMap<Long, String> mapWithinRange = getBlobsWithinRange(pages);

    //Download only for keys within that range
    for(Map.Entry<Long, String> blobMeta : mapWithinRange.entrySet()) {
        log.info("Downloading file: " + blobMeta.getValue());
        downloadWorkers.submit(new BlobDownload(downloadDir, store, container, blobMeta.getValue()));
        lastMarker = blobMeta.getValue();
        synchronized (CloudFilesManager.this) {
            // this is where we resume from.
            MarkerUtils.writeLastMarker(blobMeta.getValue());
        }
    }
    log.info("Updated the last marker value as " + lastMarker);
}
 

@Override
protected void doCall(JcloudsLocation loc, String indent) throws Exception {
    BlobStoreContext context = BlobStoreContextFactoryImpl.INSTANCE.newBlobStoreContext(loc);
    try {
        org.jclouds.blobstore.BlobStore blobStore = context.getBlobStore();
        doCall(blobStore, indent);
    } finally {
        context.close();
    }
}
 

public File copyFileFromBucket(
    String blobFileName, String bucketName, String accessId, String secretKey ) throws Exception {

    // setup to use JCloud BlobStore interface to AWS S3

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(
        new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(),
        new NettyPayloadModule());

    BlobStoreContext context = ContextBuilder.newBuilder("s3")
        .credentials(accessId, secretKey)
        .modules(MODULES)
        .overrides(overrides)
        .buildView(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();

    // get file from configured bucket, copy it to local temp file

    Blob blob = blobStore.getBlob(bucketName, blobFileName);
    if ( blob == null) {
        throw new RuntimeException(
            "Blob file name " + blobFileName + " not found in bucket " + bucketName );
    }

    FileOutputStream fop = null;
    File tempFile;
    try {
        tempFile = File.createTempFile(bucketName, RandomStringUtils.randomAlphabetic(10));
        tempFile.deleteOnExit();
        fop = new FileOutputStream(tempFile);
        InputStream is = blob.getPayload().openStream();
        IOUtils.copyLarge(is, fop);
        return tempFile;

    } finally {
        if ( fop != null ) {
            fop.close();
        }
    }
}
 

@Override
public List<String> getBucketFileNames(
    String bucketName, String endsWith, String accessId, String secretKey ) {

    // get setup to use JCloud BlobStore interface to AWS S3

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(
        new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(),
        new NettyPayloadModule());

    BlobStoreContext context = ContextBuilder.newBuilder("s3")
        .credentials(accessId, secretKey)
        .modules(MODULES)
        .overrides(overrides)
        .buildView(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();

    // gets all the files in the configured bucket recursively

    PageSet<? extends StorageMetadata> pageSets =
        blobStore.list(bucketName, new ListContainerOptions().recursive());

    if (logger.isTraceEnabled()) {
        logger.trace("   Found {} files in bucket {}", pageSets.size(), bucketName);
    }

    List<String> blobFileNames = new ArrayList<>();
    for ( Object pageSet : pageSets ) {
        String blobFileName = ((MutableBlobMetadata)pageSet).getName();
        if ( blobFileName.endsWith( endsWith )) {
            blobFileNames.add(blobFileName);
        }
    }

    return blobFileNames;
}
 

static S3ProxyLaunchInfo startS3Proxy(String configFile) throws Exception {
    S3ProxyLaunchInfo info = new S3ProxyLaunchInfo();

    try (InputStream is = Resources.asByteSource(Resources.getResource(
            configFile)).openStream()) {
        info.getProperties().load(is);
    }

    String provider = info.getProperties().getProperty(
            Constants.PROPERTY_PROVIDER);
    String identity = info.getProperties().getProperty(
            Constants.PROPERTY_IDENTITY);
    String credential = info.getProperties().getProperty(
            Constants.PROPERTY_CREDENTIAL);
    if (provider.equals("google-cloud-storage")) {
        File credentialFile = new File(credential);
        if (credentialFile.exists()) {
            credential = Files.asCharSource(credentialFile,
                    StandardCharsets.UTF_8).read();
        }
        info.getProperties().remove(Constants.PROPERTY_CREDENTIAL);
    }
    String endpoint = info.getProperties().getProperty(
            Constants.PROPERTY_ENDPOINT);

    ContextBuilder builder = ContextBuilder
            .newBuilder(provider)
            .credentials(identity, credential)
            .modules(ImmutableList.<Module>of(new SLF4JLoggingModule()))
            .overrides(info.getProperties());
    if (!Strings.isNullOrEmpty(endpoint)) {
        builder.endpoint(endpoint);
    }
    BlobStoreContext context = builder.build(BlobStoreContext.class);
    info.blobStore = context.getBlobStore();

    S3Proxy.Builder s3ProxyBuilder = S3Proxy.Builder.fromProperties(
            info.getProperties());
    s3ProxyBuilder.blobStore(info.blobStore);
    info.endpoint = s3ProxyBuilder.getEndpoint();
    info.secureEndpoint = s3ProxyBuilder.getSecureEndpoint();
    info.s3Identity = s3ProxyBuilder.getIdentity();
    info.s3Credential = s3ProxyBuilder.getCredential();
    info.servicePath = s3ProxyBuilder.getServicePath();
    info.getProperties().setProperty(Constants.PROPERTY_USER_AGENT,
            String.format("s3proxy/%s jclouds/%s java/%s",
                    TestUtils.class.getPackage().getImplementationVersion(),
                    JcloudsVersion.get(),
                    System.getProperty("java.version")));

    // resolve relative path for tests
    String keyStorePath = info.getProperties().getProperty(
            S3ProxyConstants.PROPERTY_KEYSTORE_PATH);
    String keyStorePassword = info.getProperties().getProperty(
            S3ProxyConstants.PROPERTY_KEYSTORE_PASSWORD);
    if (keyStorePath != null || keyStorePassword != null) {
        s3ProxyBuilder.keyStore(
                Resources.getResource(keyStorePath).toString(),
                keyStorePassword);
    }
    info.s3Proxy = s3ProxyBuilder.build();
    info.s3Proxy.start();
    while (!info.s3Proxy.getState().equals(AbstractLifeCycle.STARTED)) {
        Thread.sleep(1);
    }

    // reset endpoint to handle zero port
    info.endpoint = new URI(info.endpoint.getScheme(),
            info.endpoint.getUserInfo(), info.endpoint.getHost(),
            info.s3Proxy.getPort(), info.endpoint.getPath(),
            info.endpoint.getQuery(), info.endpoint.getFragment());
    if (info.secureEndpoint != null) {
        info.secureEndpoint = new URI(info.secureEndpoint.getScheme(),
                info.secureEndpoint.getUserInfo(),
                info.secureEndpoint.getHost(),
                info.s3Proxy.getSecurePort(),
                info.secureEndpoint.getPath(),
                info.secureEndpoint.getQuery(),
                info.secureEndpoint.getFragment());
    }

    return info;
}
 

private static BlobStore createBlobStore(Properties properties,
        ExecutorService executorService) throws IOException {
    String provider = properties.getProperty(Constants.PROPERTY_PROVIDER);
    String identity = properties.getProperty(Constants.PROPERTY_IDENTITY);
    String credential = properties.getProperty(
            Constants.PROPERTY_CREDENTIAL);
    String endpoint = properties.getProperty(Constants.PROPERTY_ENDPOINT);
    properties.remove(Constants.PROPERTY_ENDPOINT);
    String region = properties.getProperty(
            LocationConstants.PROPERTY_REGION);

    if (provider == null) {
        System.err.println(
                "Properties file must contain: " +
                Constants.PROPERTY_PROVIDER);
        System.exit(1);
    }

    if (provider.equals("filesystem") || provider.equals("transient")) {
        // local blobstores do not require credentials
        identity = Strings.nullToEmpty(identity);
        credential = Strings.nullToEmpty(credential);
    } else if (provider.equals("google-cloud-storage")) {
        File credentialFile = new File(credential);
        if (credentialFile.exists()) {
            credential = Files.asCharSource(credentialFile,
                    StandardCharsets.UTF_8).read();
        }
        properties.remove(Constants.PROPERTY_CREDENTIAL);
    }

    if (identity == null || credential == null) {
        System.err.println(
                "Properties file must contain: " +
                Constants.PROPERTY_IDENTITY + " and " +
                Constants.PROPERTY_CREDENTIAL);
        System.exit(1);
    }

    properties.setProperty(Constants.PROPERTY_USER_AGENT,
            String.format("s3proxy/%s jclouds/%s java/%s",
                    Main.class.getPackage().getImplementationVersion(),
                    JcloudsVersion.get(),
                    System.getProperty("java.version")));

    ContextBuilder builder = ContextBuilder
            .newBuilder(provider)
            .credentials(identity, credential)
            .modules(ImmutableList.<Module>of(
                    new SLF4JLoggingModule(),
                    new ExecutorServiceModule(executorService)))
            .overrides(properties);
    if (!Strings.isNullOrEmpty(endpoint)) {
        builder = builder.endpoint(endpoint);
    }

    BlobStoreContext context = builder.build(BlobStoreContext.class);
    BlobStore blobStore;
    if (context instanceof RegionScopedBlobStoreContext &&
            region != null) {
        blobStore = ((RegionScopedBlobStoreContext) context)
                .getBlobStore(region);
    } else {
        blobStore = context.getBlobStore();
    }
    return blobStore;
}
 

@Test
@Ignore("Pending merge of export-feature branch")
public void testIntegration100EntitiesOn() throws Exception {

    if (logger.isDebugEnabled()) {
        logger.debug("testIntegration100EntitiesOn(): starting...");
    }

    ExportService exportService = setup.getExportService();

    String appName = newOrgAppAdminRule.getApplicationInfo().getName();
    HashMap<String, Object> payload = payloadBuilder(appName);

    payload.put( "organizationId", organization.getUuid() );
    payload.put( "applicationId", applicationId );

    // create five applications each with collection of five entities

    for ( int i = 0; i < 5; i++ ) {

        ApplicationInfo appMade = setup.getMgmtSvc().createApplication( organization.getUuid(), "superapp" + i );
        EntityManager appEm = setup.getEmf().getEntityManager( appMade.getId() );

        String collName = "superappCol" + i;
        appEm.createApplicationCollection(collName);

        Map<String, Object> entityLevelProperties = null;
        Entity[] entNotCopied;
        entNotCopied = new Entity[5];

        for ( int index = 0; index < 5; index++ ) {
            entityLevelProperties = new LinkedHashMap<String, Object>();
            entityLevelProperties.put( "username", "bobso" + index );
            entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
            entNotCopied[index] = appEm.create( collName, entityLevelProperties );
        }
    }

    // export the organization containing those apps and collections

    UUID exportUUID = exportService.schedule( payload );

    int maxRetries = 100;
    int retries = 0;
    while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) && retries++ < maxRetries ) {
        Thread.sleep(100);
    }

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
    Properties overrides = new Properties();
    overrides.setProperty( "s3" + ".identity", accessId );
    overrides.setProperty( "s3" + ".credential", secretKey );

    // test that we can find the file that were exported to S3

    BlobStore blobStore = null;
    try {

        final Iterable<? extends Module> MODULES = ImmutableSet.of(
            new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(),
            new NettyPayloadModule());

        BlobStoreContext context = ContextBuilder.newBuilder("s3")
            .credentials(accessId, secretKey)
            .modules(MODULES)
            .overrides(overrides)
            .buildView(BlobStoreContext.class);

        String expectedFileName = ((ExportServiceImpl) exportService)
            .prepareOutputFileName(organization.getName(), "applications");

        blobStore = context.getBlobStore();
        if (!blobStore.blobExists(bucketName, expectedFileName)) {
            blobStore.deleteContainer(bucketName);
            Assert.fail("Blob does not exist: " + expectedFileName);
        }
        Blob bo = blobStore.getBlob(bucketName, expectedFileName);

        Long numOfFiles = blobStore.countBlobs(bucketName);
        Long numWeWant = 1L;
        blobStore.deleteContainer(bucketName);
        assertEquals(numOfFiles, numWeWant);
        assertNotNull(bo);

    } finally {
        blobStore.deleteContainer(bucketName);
    }
}
 

@Test
@Ignore("Pending merge of export-feature branch")
public void testIntegration100EntitiesForAllApps() throws Exception {

    S3Export s3Export = new S3ExportImpl();
    ExportService exportService = setup.getExportService();

    String appName = newOrgAppAdminRule.getApplicationInfo().getName();
    HashMap<String, Object> payload = payloadBuilder(appName);

    OrganizationInfo orgMade = null;
    ApplicationInfo appMade = null;
    for ( int i = 0; i < 5; i++ ) {
        orgMade = setup.getMgmtSvc().createOrganization( "minorboss" + i, adminUser, true );
        for ( int j = 0; j < 5; j++ ) {
            appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp" + j );

            EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
            customMaker.createApplicationCollection( "superappCol" + j );
            //intialize user object to be posted
            Map<String, Object> entityLevelProperties = null;
            Entity[] entNotCopied;
            entNotCopied = new Entity[1];
            //creates entities
            for ( int index = 0; index < 1; index++ ) {
                entityLevelProperties = new LinkedHashMap<String, Object>();
                entityLevelProperties.put( "derp", "bacon" );
                entNotCopied[index] = customMaker.create( "superappCol" + j, entityLevelProperties );
            }
        }
    }

    payload.put( "organizationId", orgMade.getUuid() );

    UUID exportUUID = exportService.schedule( payload );
    assertNotNull( exportUUID );

    Thread.sleep( 3000 );

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

    Properties overrides = new Properties();
    overrides.setProperty( "s3" + ".identity", accessId );
    overrides.setProperty( "s3" + ".credential", secretKey );

    BlobStore blobStore = null;

    try {
        final Iterable<? extends Module> MODULES = ImmutableSet.of(
            new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(),
            new NettyPayloadModule() );

        BlobStoreContext context = ContextBuilder.newBuilder( "s3" )
            .credentials(accessId, secretKey )
            .modules(MODULES )
            .overrides(overrides )
            .buildView(BlobStoreContext.class );

        blobStore = context.getBlobStore();

        //Grab Number of files
        Long numOfFiles = blobStore.countBlobs( bucketName );

        String expectedFileName = ((ExportServiceImpl)exportService)
            .prepareOutputFileName(organization.getName(), "applications");

        //delete container containing said files
        Blob bo = blobStore.getBlob(bucketName, expectedFileName);
        Long numWeWant = 5L;
        blobStore.deleteContainer( bucketName );

        //asserts that the correct number of files was transferred over
        assertEquals( numWeWant, numOfFiles );

    }
    finally {
        blobStore.deleteContainer( bucketName );
    }
}
 

@Test
@Ignore("Pending merge of export-feature branch")
public void testIntegration100EntitiesOnOneOrg() throws Exception {

    S3Export s3Export = new S3ExportImpl();
    ExportService exportService = setup.getExportService();

    String appName = newOrgAppAdminRule.getApplicationInfo().getName();
    HashMap<String, Object> payload = payloadBuilder(appName);

    payload.put( "organizationId", organization.getUuid() );
    payload.put( "applicationId", applicationId );

    OrganizationInfo orgMade = null;
    ApplicationInfo appMade = null;
    for ( int i = 0; i < 100; i++ ) {
        orgMade = setup.getMgmtSvc().createOrganization( "largerboss" + i, adminUser, true );
        appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp" + i );

        EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
        customMaker.createApplicationCollection( "superappCol" + i );
        //intialize user object to be posted
        Map<String, Object> entityLevelProperties = null;
        Entity[] entNotCopied;
        entNotCopied = new Entity[20];
        //creates entities
        for ( int index = 0; index < 20; index++ ) {
            entityLevelProperties = new LinkedHashMap<String, Object>();
            entityLevelProperties.put( "username", "bobso" + index );
            entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
            entNotCopied[index] = customMaker.create( "superappCol", entityLevelProperties );
        }
    }

    EntityManager em = setup.getEmf().getEntityManager( applicationId );

    //intialize user object to be posted
    Map<String, Object> userProperties = null;
    Entity[] entity;
    entity = new Entity[100];

    //creates entities
    for ( int i = 0; i < 100; i++ ) {
        userProperties = new LinkedHashMap<String, Object>();
        userProperties.put( "username", "bido" + i );
        userProperties.put( "email", "bido" + i + "@anuff.com" );

        entity[i] = em.create( "user", userProperties );
    }

    UUID exportUUID = exportService.schedule( payload );

    while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) ) {}

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

    Properties overrides = new Properties();
    overrides.setProperty( "s3" + ".identity", accessId );
    overrides.setProperty( "s3" + ".credential", secretKey );

    Blob bo = null;
    BlobStore blobStore = null;

    try {
        final Iterable<? extends Module> MODULES = ImmutableSet.of( new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(), new NettyPayloadModule() );

        BlobStoreContext context = ContextBuilder.newBuilder( "s3" )
            .credentials( accessId, secretKey )
            .modules( MODULES )
            .overrides( overrides )
            .buildView( BlobStoreContext.class );

        String expectedFileName = ((ExportServiceImpl)exportService)
            .prepareOutputFileName(organization.getName(), "applications");

        blobStore = context.getBlobStore();
        if ( !blobStore.blobExists( bucketName, expectedFileName ) ) {
            assert ( false );
        }
        Long numOfFiles = blobStore.countBlobs( bucketName );
        Long numWeWant = Long.valueOf( 1 );
        assertEquals( numOfFiles, numWeWant );

        bo = blobStore.getBlob( bucketName, expectedFileName );
    }
    catch ( Exception e ) {
        assert ( false );
    }

    assertNotNull( bo );
    blobStore.deleteContainer( bucketName );
}
 

private ObjectStoreFileStorage createObjectStoreFileStorage() {
    BlobStoreContext context = getBlobStoreContext();
    return context == null ? null : new ObjectStoreFileStorage(context.getBlobStore(), getServiceInfo().getContainer());
}