Java Code Examples for org.jclouds.blobstore.BlobStore#list()

The following examples show how to use org.jclouds.blobstore.BlobStore#list() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CloudFilesManager.java    From blueflood with Apache License 2.0 6 votes vote down vote up 
public synchronized boolean hasNewFiles() {
    // see if there are any files since lastMarker.
    BlobStoreContext ctx = ContextBuilder.newBuilder(provider)
            .credentials(user, key)
            .overrides(new Properties() {{
                setProperty(LocationConstants.PROPERTY_ZONE, zone);
            }})
            .buildView(BlobStoreContext.class);
    
    BlobStore store = ctx.getBlobStore();
    ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker);
    PageSet<? extends StorageMetadata> pages = store.list(container, options);
    
    log.debug("Saw {} new files since {}", pages.size() == batchSize ? "many" : Integer.toString(pages.size()), lastMarker);
    boolean emptiness = getBlobsWithinRange(pages).isEmpty();

    if(emptiness) {
        log.warn("No file found within range {}", new Range(START_TIME, STOP_TIME));
    } else {
        log.debug("New files found within range {}", new Range(START_TIME, STOP_TIME));
    }

    return !emptiness;
}
Example 2
Source File: CloudExplorerSupport.java    From brooklyn-server with Apache License 2.0 5 votes vote down vote up 
@Override
protected void doCall(BlobStore blobstore, String indent) throws Exception {
    Set<? extends StorageMetadata> containers = blobstore.list();
    stdout.println(indent+"Containers {");
    for (StorageMetadata container : containers) {
        stdout.println(indent+"\t"+container);
    }
    stdout.println(indent+"}");
}
Example 3
Source File: CloudExplorerSupport.java    From brooklyn-server with Apache License 2.0 5 votes vote down vote up 
@Override
protected void doCall(BlobStore blobStore, String indent) throws Exception {
    for (String containerName : names) {
        Set<? extends StorageMetadata> contents = blobStore.list(containerName);
        stdout.println(indent+"Container "+containerName+" {");
        for (StorageMetadata content : contents) {
            stdout.println(indent+"\t"+content);
        }
        stdout.println(indent+"}");
    }
}
Example 4
Source File: S3ProxyRule.java    From s3proxy with Apache License 2.0 5 votes vote down vote up 
@Override
protected void after() {
    logger.debug("S3 proxy is stopping");
    try {
        s3Proxy.stop();
        BlobStore blobStore = blobStoreContext.getBlobStore();
        for (StorageMetadata metadata : blobStore.list()) {
            blobStore.deleteContainer(metadata.getName());
        }
        blobStoreContext.close();
    } catch (Exception e) {
        throw new RuntimeException("Unable to stop S3 proxy", e);
    }
    FileUtils.deleteQuietly(blobStoreLocation);
    logger.debug("S3 proxy has stopped");
}
Example 5
Source File: CloudFilesManager.java    From blueflood with Apache License 2.0 5 votes vote down vote up 
public synchronized void downloadNewFiles(File downloadDir) {
    log.info("Downloading new files since {}", lastMarker);
    
    BlobStoreContext ctx = ContextBuilder.newBuilder(provider)
            .credentials(user, key)
            .overrides(new Properties() {{
                setProperty(LocationConstants.PROPERTY_ZONE, zone);
            }})
            .buildView(BlobStoreContext.class);

    // threadsafe according to https://jclouds.apache.org/documentation/userguide/blobstore-guide/
    BlobStore store = ctx.getBlobStore();
    ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker);
    PageSet<? extends StorageMetadata> pages = store.list(container, options);

    //Gets key within the time range specified
    NavigableMap<Long, String> mapWithinRange = getBlobsWithinRange(pages);

    //Download only for keys within that range
    for(Map.Entry<Long, String> blobMeta : mapWithinRange.entrySet()) {
        log.info("Downloading file: " + blobMeta.getValue());
        downloadWorkers.submit(new BlobDownload(downloadDir, store, container, blobMeta.getValue()));
        lastMarker = blobMeta.getValue();
        synchronized (CloudFilesManager.this) {
            // this is where we resume from.
            MarkerUtils.writeLastMarker(blobMeta.getValue());
        }
    }
    log.info("Updated the last marker value as " + lastMarker);
}
Example 6
Source File: ImportCollectionIT.java    From usergrid with Apache License 2.0 5 votes vote down vote up 
private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix );

        String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
        String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for ( Object o : blobStoreList.toArray() ) {
            StorageMetadata s = (StorageMetadata)o;

            if ( s.getName().startsWith( bucketPrefix )) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch ( ContainerNotFoundException cnfe ) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe );
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
Example 7
Source File: ImportResourceIT.java    From usergrid with Apache License 2.0 5 votes vote down vote up 
private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix);

        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides ).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for (Object o : blobStoreList.toArray()) {
            StorageMetadata s = (StorageMetadata) o;

            if (s.getName().startsWith(bucketPrefix)) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch (ContainerNotFoundException cnfe) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe);
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
Example 8
Source File: S3ImportImpl.java    From usergrid with Apache License 2.0 4 votes vote down vote up 
@Override
public List<String> getBucketFileNames(
    String bucketName, String endsWith, String accessId, String secretKey ) {

    // get setup to use JCloud BlobStore interface to AWS S3

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(
        new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(),
        new NettyPayloadModule());

    BlobStoreContext context = ContextBuilder.newBuilder("s3")
        .credentials(accessId, secretKey)
        .modules(MODULES)
        .overrides(overrides)
        .buildView(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();

    // gets all the files in the configured bucket recursively

    PageSet<? extends StorageMetadata> pageSets =
        blobStore.list(bucketName, new ListContainerOptions().recursive());

    if (logger.isTraceEnabled()) {
        logger.trace("   Found {} files in bucket {}", pageSets.size(), bucketName);
    }

    List<String> blobFileNames = new ArrayList<>();
    for ( Object pageSet : pageSets ) {
        String blobFileName = ((MutableBlobMetadata)pageSet).getName();
        if ( blobFileName.endsWith( endsWith )) {
            blobFileNames.add(blobFileName);
        }
    }

    return blobFileNames;
}