org.jclouds.logging.log4j.config.Log4JLoggingModule Java Examples

The following examples show how to use org.jclouds.logging.log4j.config.Log4JLoggingModule. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ImportCollectionIT.java    From usergrid with Apache License 2.0 6 votes vote down vote up
/**
 * Delete the configured s3 bucket.
 */
public void deleteBucket() {

    logger.debug("\n\nDelete bucket\n");

    String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
    String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet
        .of(new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(),
            new NettyPayloadModule());

    BlobStoreContext context =
        ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
            .overrides(overrides).buildView(BlobStoreContext.class);

    BlobStore blobStore = context.getBlobStore();
    blobStore.deleteContainer( bucketName );
}
 
Example #2
Source File: ImportServiceIT.java    From usergrid with Apache License 2.0 6 votes vote down vote up
public void deleteBucket() {

        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

        Properties overrides = new Properties();
        overrides.setProperty( "s3" + ".identity", accessId );
        overrides.setProperty( "s3" + ".credential", secretKey );

        Blob bo = null;
        BlobStore blobStore = null;
        final Iterable<? extends Module> MODULES = ImmutableSet
                .of(new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
                        new NettyPayloadModule());

        BlobStoreContext context =
                ContextBuilder.newBuilder("s3").credentials( accessId, secretKey ).modules( MODULES )
                        .overrides( overrides ).buildView( BlobStoreContext.class );

        blobStore = context.getBlobStore();
        blobStore.deleteContainer( bucketName );
    }
 
Example #3
Source File: ImportResourceIT.java    From usergrid with Apache License 2.0 6 votes vote down vote up
/**
 * Delete the configured s3 bucket.
 */
public void deleteBucket() {

    logger.debug("\n\nDelete bucket\n");

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(), new NettyPayloadModule());

    BlobStoreContext context =
        ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
            .overrides(overrides ).buildView(BlobStoreContext.class);

    BlobStore blobStore = context.getBlobStore();
    blobStore.deleteContainer(bucketName);
}
 
Example #4
Source File: WarehouseExport.java    From usergrid with Apache License 2.0 6 votes vote down vote up
private void copyToS3( String fileName ) {

        String bucketName = ( String ) properties.get( BUCKET_PROPNAME );
        String accessId = ( String ) properties.get( ACCESS_ID_PROPNAME );
        String secretKey = ( String ) properties.get( SECRET_KEY_PROPNAME );

        Properties overrides = new Properties();
        overrides.setProperty( "s3" + ".identity", accessId );
        overrides.setProperty( "s3" + ".credential", secretKey );

        final Iterable<? extends Module> MODULES = ImmutableSet
                .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
                        new NettyPayloadModule() );

        AWSCredentials credentials = new BasicAWSCredentials(accessId, secretKey);
        ClientConfiguration clientConfig = new ClientConfiguration();
        clientConfig.setProtocol( Protocol.HTTP);

        AmazonS3Client s3Client = new AmazonS3Client(credentials, clientConfig);

        s3Client.createBucket( bucketName );
        File uploadFile = new File( fileName );
        PutObjectResult putObjectResult = s3Client.putObject( bucketName, uploadFile.getName(), uploadFile );
        logger.info("Uploaded file etag={}", putObjectResult.getETag());
    }
 
Example #5
Source File: ImportCollectionIT.java    From usergrid with Apache License 2.0 5 votes vote down vote up
private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix );

        String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
        String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for ( Object o : blobStoreList.toArray() ) {
            StorageMetadata s = (StorageMetadata)o;

            if ( s.getName().startsWith( bucketPrefix )) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch ( ContainerNotFoundException cnfe ) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe );
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
 
Example #6
Source File: ImportResourceIT.java    From usergrid with Apache License 2.0 5 votes vote down vote up
private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix);

        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides ).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for (Object o : blobStoreList.toArray()) {
            StorageMetadata s = (StorageMetadata) o;

            if (s.getName().startsWith(bucketPrefix)) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch (ContainerNotFoundException cnfe) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe);
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
 
Example #7
Source File: S3ImportImpl.java    From usergrid with Apache License 2.0 4 votes vote down vote up
public File copyFileFromBucket(
    String blobFileName, String bucketName, String accessId, String secretKey ) throws Exception {

    // setup to use JCloud BlobStore interface to AWS S3

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(
        new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(),
        new NettyPayloadModule());

    BlobStoreContext context = ContextBuilder.newBuilder("s3")
        .credentials(accessId, secretKey)
        .modules(MODULES)
        .overrides(overrides)
        .buildView(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();

    // get file from configured bucket, copy it to local temp file

    Blob blob = blobStore.getBlob(bucketName, blobFileName);
    if ( blob == null) {
        throw new RuntimeException(
            "Blob file name " + blobFileName + " not found in bucket " + bucketName );
    }

    FileOutputStream fop = null;
    File tempFile;
    try {
        tempFile = File.createTempFile(bucketName, RandomStringUtils.randomAlphabetic(10));
        tempFile.deleteOnExit();
        fop = new FileOutputStream(tempFile);
        InputStream is = blob.getPayload().openStream();
        IOUtils.copyLarge(is, fop);
        return tempFile;

    } finally {
        if ( fop != null ) {
            fop.close();
        }
    }
}
 
Example #8
Source File: S3ImportImpl.java    From usergrid with Apache License 2.0 4 votes vote down vote up
@Override
public List<String> getBucketFileNames(
    String bucketName, String endsWith, String accessId, String secretKey ) {

    // get setup to use JCloud BlobStore interface to AWS S3

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(
        new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(),
        new NettyPayloadModule());

    BlobStoreContext context = ContextBuilder.newBuilder("s3")
        .credentials(accessId, secretKey)
        .modules(MODULES)
        .overrides(overrides)
        .buildView(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();

    // gets all the files in the configured bucket recursively

    PageSet<? extends StorageMetadata> pageSets =
        blobStore.list(bucketName, new ListContainerOptions().recursive());

    if (logger.isTraceEnabled()) {
        logger.trace("   Found {} files in bucket {}", pageSets.size(), bucketName);
    }

    List<String> blobFileNames = new ArrayList<>();
    for ( Object pageSet : pageSets ) {
        String blobFileName = ((MutableBlobMetadata)pageSet).getName();
        if ( blobFileName.endsWith( endsWith )) {
            blobFileNames.add(blobFileName);
        }
    }

    return blobFileNames;
}
 
Example #9
Source File: ExportServiceIT.java    From usergrid with Apache License 2.0 4 votes vote down vote up
@Test
@Ignore("Pending merge of export-feature branch")
public void testIntegration100EntitiesOn() throws Exception {

    if (logger.isDebugEnabled()) {
        logger.debug("testIntegration100EntitiesOn(): starting...");
    }

    ExportService exportService = setup.getExportService();

    String appName = newOrgAppAdminRule.getApplicationInfo().getName();
    HashMap<String, Object> payload = payloadBuilder(appName);

    payload.put( "organizationId", organization.getUuid() );
    payload.put( "applicationId", applicationId );

    // create five applications each with collection of five entities

    for ( int i = 0; i < 5; i++ ) {

        ApplicationInfo appMade = setup.getMgmtSvc().createApplication( organization.getUuid(), "superapp" + i );
        EntityManager appEm = setup.getEmf().getEntityManager( appMade.getId() );

        String collName = "superappCol" + i;
        appEm.createApplicationCollection(collName);

        Map<String, Object> entityLevelProperties = null;
        Entity[] entNotCopied;
        entNotCopied = new Entity[5];

        for ( int index = 0; index < 5; index++ ) {
            entityLevelProperties = new LinkedHashMap<String, Object>();
            entityLevelProperties.put( "username", "bobso" + index );
            entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
            entNotCopied[index] = appEm.create( collName, entityLevelProperties );
        }
    }

    // export the organization containing those apps and collections

    UUID exportUUID = exportService.schedule( payload );

    int maxRetries = 100;
    int retries = 0;
    while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) && retries++ < maxRetries ) {
        Thread.sleep(100);
    }

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
    Properties overrides = new Properties();
    overrides.setProperty( "s3" + ".identity", accessId );
    overrides.setProperty( "s3" + ".credential", secretKey );

    // test that we can find the file that were exported to S3

    BlobStore blobStore = null;
    try {

        final Iterable<? extends Module> MODULES = ImmutableSet.of(
            new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(),
            new NettyPayloadModule());

        BlobStoreContext context = ContextBuilder.newBuilder("s3")
            .credentials(accessId, secretKey)
            .modules(MODULES)
            .overrides(overrides)
            .buildView(BlobStoreContext.class);

        String expectedFileName = ((ExportServiceImpl) exportService)
            .prepareOutputFileName(organization.getName(), "applications");

        blobStore = context.getBlobStore();
        if (!blobStore.blobExists(bucketName, expectedFileName)) {
            blobStore.deleteContainer(bucketName);
            Assert.fail("Blob does not exist: " + expectedFileName);
        }
        Blob bo = blobStore.getBlob(bucketName, expectedFileName);

        Long numOfFiles = blobStore.countBlobs(bucketName);
        Long numWeWant = 1L;
        blobStore.deleteContainer(bucketName);
        assertEquals(numOfFiles, numWeWant);
        assertNotNull(bo);

    } finally {
        blobStore.deleteContainer(bucketName);
    }
}
 
Example #10
Source File: ExportServiceIT.java    From usergrid with Apache License 2.0 4 votes vote down vote up
@Test
@Ignore("Pending merge of export-feature branch")
public void testIntegration100EntitiesForAllApps() throws Exception {

    S3Export s3Export = new S3ExportImpl();
    ExportService exportService = setup.getExportService();

    String appName = newOrgAppAdminRule.getApplicationInfo().getName();
    HashMap<String, Object> payload = payloadBuilder(appName);

    OrganizationInfo orgMade = null;
    ApplicationInfo appMade = null;
    for ( int i = 0; i < 5; i++ ) {
        orgMade = setup.getMgmtSvc().createOrganization( "minorboss" + i, adminUser, true );
        for ( int j = 0; j < 5; j++ ) {
            appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp" + j );

            EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
            customMaker.createApplicationCollection( "superappCol" + j );
            //intialize user object to be posted
            Map<String, Object> entityLevelProperties = null;
            Entity[] entNotCopied;
            entNotCopied = new Entity[1];
            //creates entities
            for ( int index = 0; index < 1; index++ ) {
                entityLevelProperties = new LinkedHashMap<String, Object>();
                entityLevelProperties.put( "derp", "bacon" );
                entNotCopied[index] = customMaker.create( "superappCol" + j, entityLevelProperties );
            }
        }
    }

    payload.put( "organizationId", orgMade.getUuid() );

    UUID exportUUID = exportService.schedule( payload );
    assertNotNull( exportUUID );

    Thread.sleep( 3000 );

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

    Properties overrides = new Properties();
    overrides.setProperty( "s3" + ".identity", accessId );
    overrides.setProperty( "s3" + ".credential", secretKey );

    BlobStore blobStore = null;

    try {
        final Iterable<? extends Module> MODULES = ImmutableSet.of(
            new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(),
            new NettyPayloadModule() );

        BlobStoreContext context = ContextBuilder.newBuilder( "s3" )
            .credentials(accessId, secretKey )
            .modules(MODULES )
            .overrides(overrides )
            .buildView(BlobStoreContext.class );

        blobStore = context.getBlobStore();

        //Grab Number of files
        Long numOfFiles = blobStore.countBlobs( bucketName );

        String expectedFileName = ((ExportServiceImpl)exportService)
            .prepareOutputFileName(organization.getName(), "applications");

        //delete container containing said files
        Blob bo = blobStore.getBlob(bucketName, expectedFileName);
        Long numWeWant = 5L;
        blobStore.deleteContainer( bucketName );

        //asserts that the correct number of files was transferred over
        assertEquals( numWeWant, numOfFiles );

    }
    finally {
        blobStore.deleteContainer( bucketName );
    }
}
 
Example #11
Source File: ExportServiceIT.java    From usergrid with Apache License 2.0 4 votes vote down vote up
@Test
@Ignore("Pending merge of export-feature branch")
public void testIntegration100EntitiesOnOneOrg() throws Exception {

    S3Export s3Export = new S3ExportImpl();
    ExportService exportService = setup.getExportService();

    String appName = newOrgAppAdminRule.getApplicationInfo().getName();
    HashMap<String, Object> payload = payloadBuilder(appName);

    payload.put( "organizationId", organization.getUuid() );
    payload.put( "applicationId", applicationId );

    OrganizationInfo orgMade = null;
    ApplicationInfo appMade = null;
    for ( int i = 0; i < 100; i++ ) {
        orgMade = setup.getMgmtSvc().createOrganization( "largerboss" + i, adminUser, true );
        appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp" + i );

        EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
        customMaker.createApplicationCollection( "superappCol" + i );
        //intialize user object to be posted
        Map<String, Object> entityLevelProperties = null;
        Entity[] entNotCopied;
        entNotCopied = new Entity[20];
        //creates entities
        for ( int index = 0; index < 20; index++ ) {
            entityLevelProperties = new LinkedHashMap<String, Object>();
            entityLevelProperties.put( "username", "bobso" + index );
            entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
            entNotCopied[index] = customMaker.create( "superappCol", entityLevelProperties );
        }
    }

    EntityManager em = setup.getEmf().getEntityManager( applicationId );

    //intialize user object to be posted
    Map<String, Object> userProperties = null;
    Entity[] entity;
    entity = new Entity[100];

    //creates entities
    for ( int i = 0; i < 100; i++ ) {
        userProperties = new LinkedHashMap<String, Object>();
        userProperties.put( "username", "bido" + i );
        userProperties.put( "email", "bido" + i + "@anuff.com" );

        entity[i] = em.create( "user", userProperties );
    }

    UUID exportUUID = exportService.schedule( payload );

    while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) ) {}

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

    Properties overrides = new Properties();
    overrides.setProperty( "s3" + ".identity", accessId );
    overrides.setProperty( "s3" + ".credential", secretKey );

    Blob bo = null;
    BlobStore blobStore = null;

    try {
        final Iterable<? extends Module> MODULES = ImmutableSet.of( new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(), new NettyPayloadModule() );

        BlobStoreContext context = ContextBuilder.newBuilder( "s3" )
            .credentials( accessId, secretKey )
            .modules( MODULES )
            .overrides( overrides )
            .buildView( BlobStoreContext.class );

        String expectedFileName = ((ExportServiceImpl)exportService)
            .prepareOutputFileName(organization.getName(), "applications");

        blobStore = context.getBlobStore();
        if ( !blobStore.blobExists( bucketName, expectedFileName ) ) {
            assert ( false );
        }
        Long numOfFiles = blobStore.countBlobs( bucketName );
        Long numWeWant = Long.valueOf( 1 );
        assertEquals( numOfFiles, numWeWant );

        bo = blobStore.getBlob( bucketName, expectedFileName );
    }
    catch ( Exception e ) {
        assert ( false );
    }

    assertNotNull( bo );
    blobStore.deleteContainer( bucketName );
}