org.jclouds.blobstore.options.ListContainerOptions Java Examples

@Test
public void testList() throws Exception {
    ObjectListing objectListing = new ObjectListing();
    objectListing.setBucketName(CONTAINER);
    objectListing.setObjectSummaries(getObjectSummaries(3));
    Mockito.when(ossClient.listObjects(any(ListObjectsRequest.class)))
           .thenReturn(objectListing);

    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setUserMetadata(getUserMetadata());

    Mockito.when(ossClient.getObjectMetadata(any(String.class), any(String.class)))
           .thenReturn(objectMetadata);
    Mockito.when(ossClient.generatePresignedUrl(any(), any(), any()))
           .thenReturn(new URL("https://oss-eu-central-1.aliyuncs.com"));
    aliOSSBlobStore.list(CONTAINER, new ListContainerOptions().withDetails())
                   .forEach(storageMetadata -> {
        assertTrue(storageMetadata.getName()
                                  .startsWith(FILENAME));
        assertEquals(PAYLOAD.length(), storageMetadata.getSize());
        assertTrue(storageMetadata.getETag()
                                  .startsWith(FILENAME));
        assertEquals(getUserMetadata(), storageMetadata.getUserMetadata());
    });
}
 

public void testCreateListDestroySimpleDirInContainer() throws IOException {
    context.getBlobStore().createContainerInLocation(null, testContainerName);
    context.getBlobStore().createDirectory(testContainerName, "my-dir-1");
    
    PageSet<? extends StorageMetadata> ps = context.getBlobStore().list(testContainerName);
    assertHasItemNamed(ps, "my-dir-1");
    
    Blob b = context.getBlobStore().blobBuilder("my-blob-1").payload(Streams.newInputStreamWithContents("hello world")).build();
    context.getBlobStore().putBlob(testContainerName+"/"+"my-dir-1", b);
    
    ps = context.getBlobStore().list(testContainerName, ListContainerOptions.Builder.inDirectory("my-dir-1"));
    assertHasItemNamed(ps, "my-dir-1/my-blob-1");

    // both these syntaxes work:
    Blob b2 = context.getBlobStore().getBlob(testContainerName+"/"+"my-dir-1", "my-blob-1");
    Assert.assertEquals(Streams.readFullyStringAndClose(b2.getPayload().openStream()), "hello world");

    Blob b3 = context.getBlobStore().getBlob(testContainerName, "my-dir-1"+"/"+"my-blob-1");
    Assert.assertEquals(Streams.readFullyStringAndClose(b3.getPayload().openStream()), "hello world");

    context.getBlobStore().deleteContainer(testContainerName);
}
 

private Set<String> listAllBlobs() {
    Set<String> blobNames = new HashSet<String>();
    ListContainerOptions options = new ListContainerOptions();
    while (true) {
        PageSet<? extends StorageMetadata> set = blobStoreRead.list(
                containerName, options);
        for (StorageMetadata sm : set) {
            blobNames.add(sm.getName());
        }
        String marker = set.getNextMarker();
        if (marker == null) {
            break;
        }
        options = options.afterMarker(marker);
    }
    return blobNames;
}
 

public synchronized boolean hasNewFiles() {
    // see if there are any files since lastMarker.
    BlobStoreContext ctx = ContextBuilder.newBuilder(provider)
            .credentials(user, key)
            .overrides(new Properties() {{
                setProperty(LocationConstants.PROPERTY_ZONE, zone);
            }})
            .buildView(BlobStoreContext.class);
    
    BlobStore store = ctx.getBlobStore();
    ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker);
    PageSet<? extends StorageMetadata> pages = store.list(container, options);
    
    log.debug("Saw {} new files since {}", pages.size() == batchSize ? "many" : Integer.toString(pages.size()), lastMarker);
    boolean emptiness = getBlobsWithinRange(pages).isEmpty();

    if(emptiness) {
        log.warn("No file found within range {}", new Range(START_TIME, STOP_TIME));
    } else {
        log.debug("New files found within range {}", new Range(START_TIME, STOP_TIME));
    }

    return !emptiness;
}
 

@Override
public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) {
    return doOssOperation(oss -> {
        ListObjectsRequest request = toListObjectRequest(container, options);
        ObjectListing objectListing = oss.listObjects(request);
        List<StorageMetadata> storageMetadataList = toStorageMetadataList(oss, container, objectListing);
        return new PageSetImpl<>(storageMetadataList, objectListing.getNextMarker());
    });
}
 

private ListObjectsRequest toListObjectRequest(String container, ListContainerOptions options) {
    ListObjectsRequest request = new ListObjectsRequest(container);
    if (options.getMaxResults() != null) {
        request.setMaxKeys(options.getMaxResults());
    }
    if (options.getMarker() != null) {
        request.setMarker(options.getMarker());
    }
    return request;
}
 

private Set<String> getEntryNames(Predicate<? super StorageMetadata> filter) {
    return blobStore.list(container, new ListContainerOptions().withDetails())
                    .stream()
                    .filter(Objects::nonNull)
                    .filter(filter)
                    .map(StorageMetadata::getName)
                    .collect(Collectors.toSet());
}
 

@Override
public List<String> listContentsWithSubPath(final String parentSubPath) {
    checkPrepared();
    return FluentIterable.from(context.getBlobStore().list(getContainerNameFirstPart(), 
        ListContainerOptions.Builder.inDirectory(getItemInContainerSubPath(parentSubPath))))
            .transform(new Function<StorageMetadata, String>() {
                @Override
                public String apply(@javax.annotation.Nullable StorageMetadata input) {
                    String result = input.getName();
                    result = Strings.removeFromStart(result, containerSubPath);
                    result = Strings.removeFromStart(result, "/");
                    return result;
                }
            }).toList();
}
 

public synchronized void downloadNewFiles(File downloadDir) {
    log.info("Downloading new files since {}", lastMarker);
    
    BlobStoreContext ctx = ContextBuilder.newBuilder(provider)
            .credentials(user, key)
            .overrides(new Properties() {{
                setProperty(LocationConstants.PROPERTY_ZONE, zone);
            }})
            .buildView(BlobStoreContext.class);

    // threadsafe according to https://jclouds.apache.org/documentation/userguide/blobstore-guide/
    BlobStore store = ctx.getBlobStore();
    ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker);
    PageSet<? extends StorageMetadata> pages = store.list(container, options);

    //Gets key within the time range specified
    NavigableMap<Long, String> mapWithinRange = getBlobsWithinRange(pages);

    //Download only for keys within that range
    for(Map.Entry<Long, String> blobMeta : mapWithinRange.entrySet()) {
        log.info("Downloading file: " + blobMeta.getValue());
        downloadWorkers.submit(new BlobDownload(downloadDir, store, container, blobMeta.getValue()));
        lastMarker = blobMeta.getValue();
        synchronized (CloudFilesManager.this) {
            // this is where we resume from.
            MarkerUtils.writeLastMarker(blobMeta.getValue());
        }
    }
    log.info("Updated the last marker value as " + lastMarker);
}
 

@Override
public List<String> getBucketFileNames(
    String bucketName, String endsWith, String accessId, String secretKey ) {

    // get setup to use JCloud BlobStore interface to AWS S3

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(
        new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(),
        new NettyPayloadModule());

    BlobStoreContext context = ContextBuilder.newBuilder("s3")
        .credentials(accessId, secretKey)
        .modules(MODULES)
        .overrides(overrides)
        .buildView(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();

    // gets all the files in the configured bucket recursively

    PageSet<? extends StorageMetadata> pageSets =
        blobStore.list(bucketName, new ListContainerOptions().recursive());

    if (logger.isTraceEnabled()) {
        logger.trace("   Found {} files in bucket {}", pageSets.size(), bucketName);
    }

    List<String> blobFileNames = new ArrayList<>();
    for ( Object pageSet : pageSets ) {
        String blobFileName = ((MutableBlobMetadata)pageSet).getName();
        if ( blobFileName.endsWith( endsWith )) {
            blobFileNames.add(blobFileName);
        }
    }

    return blobFileNames;
}