org.jclouds.blobstore.domain.Blob Java Examples

/**
 * Upload an object from a String with metadata using the BlobStore API.
 */
private void uploadObjectFromStringWithMetadata() {
   System.out.format("Upload Object From String With Metadata%n");

   String filename = "uploadObjectFromStringWithMetadata.txt";

   Map<String, String> userMetadata = new HashMap<String, String>();
   userMetadata.put("key1", "value1");

   ByteSource source = ByteSource.wrap("uploadObjectFromString".getBytes());

   Blob blob = blobStore.blobBuilder(filename)
         .payload(Payloads.newByteSourcePayload(source))
         .userMetadata(userMetadata)
         .build();

   blobStore.putBlob(CONTAINER, blob);

   System.out.format("  %s%n", filename);
}
 

public int readAfterCreate() throws IOException {
    int count = 0;
    for (int i = 0; i < iterations; ++i) {
        String blobName = makeBlobName();
        blobStore.putBlob(containerName, makeBlob(blobName, payload1));
        Blob getBlob = blobStoreRead.getBlob(containerName, blobName);
        if (getBlob == null) {
            ++count;
        } else {
            try (Payload payload = getBlob.getPayload();
                 InputStream is = payload.openStream()) {
                ByteStreams.copy(is, ByteStreams.nullOutputStream());
            }
        }
        blobStore.removeBlob(containerName, blobName);
    }
    return count;
}
 

/**
 * Upload a large object from a File using the BlobStore API.
 *
 * @throws ExecutionException
 * @throws InterruptedException
 */
private void uploadLargeObjectFromFile(File largeFile) throws InterruptedException, ExecutionException {
   System.out.format("Upload Large Object From File%n");

   ByteSource source = Files.asByteSource(largeFile);
   // create the payload and set the content length
   Payload payload = Payloads.newByteSourcePayload(source);
   payload.getContentMetadata().setContentLength(largeFile.length());

   Blob blob = blobStore.blobBuilder(largeFile.getName())
         .payload(payload)
         .build();

   // configure the blobstore to use multipart uploading of the file
   String eTag = blobStore.putBlob(CONTAINER, blob, multipart());

   System.out.format("  Uploaded %s eTag=%s", largeFile.getName(), eTag);
}
 

@Override
public InputStream getFileResourceContent( String key )
{
    final Blob blob = getBlob( key );

    if ( blob == null )
    {
        return null;
    }

    try
    {
        return blob.getPayload().openStream();
    }
    catch ( IOException e )
    {
        log.warn( String.format( "Unable to retrieve fileResource with key: %s. Message: %s", key, e.getMessage() ) );
        return null;
    }
}
 

@Override
public void addFile(FileEntry fileEntry, File file) throws FileStorageException {
    String entryName = fileEntry.getId();
    long fileSize = fileEntry.getSize()
                             .longValue();
    Blob blob = blobStore.blobBuilder(entryName)
                         .payload(file)
                         .contentDisposition(fileEntry.getName())
                         .contentType(MediaType.OCTET_STREAM.toString())
                         .userMetadata(createFileEntryMetadata(fileEntry))
                         .build();
    try {
        putBlobWithRetries(blob, 3);
        LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1_SUCCESSFULLY_2, fileEntry.getId(), fileSize));
    } catch (ContainerNotFoundException e) {
        throw new FileStorageException(MessageFormat.format(Messages.FILE_UPLOAD_FAILED, fileEntry.getName(),
                                                            fileEntry.getNamespace()));
    }
}
 

private String addBlobWithNoMetadata() throws Exception {
    BlobStore blobStore = blobStoreContext.getBlobStore();
    Path path = Paths.get(TEST_FILE_LOCATION);
    long fileSize = FileUtils.sizeOf(path.toFile());
    String id = UUID.randomUUID()
                    .toString();
    Blob blob = blobStore.blobBuilder(id)
                         .payload(new FileInputStream(path.toFile()))
                         .contentDisposition(path.getFileName()
                                                 .toString())
                         .contentType(MediaType.OCTET_STREAM.toString())
                         .contentLength(fileSize)
                         .build();
    blobStore.putBlob(CONTAINER, blob);
    return id;
}
 

public void publish(String remoteName, byte[] payload) throws IOException {
    Timer.Context ctx = uploadTimer.time();
    try {
        Blob blob = blobStore.blobBuilder(remoteName).payload(payload)
                .contentType("application/json")
                .contentEncoding(remoteName.endsWith(".gz") ? "gzip" : "identity")
                .calculateMD5().build();

        String containerName = CONTAINER_DATE_FORMAT.format(new Date());
        if (!lastContainerCreated.matches(containerName)) {
            createContainer();
        }
        blobStore.putBlob(containerName, blob);
    } finally {
        ctx.stop();
    }
}
 

@Override
public String saveFileResourceContent( FileResource fileResource, File file )
{
    Blob blob = createBlob( fileResource, StringUtils.EMPTY, file, fileResource.getContentMd5() );

    if ( blob == null )
    {
        return null;
    }

    blobStore.putBlob( config.container, blob );

    try
    {
        Files.deleteIfExists( file.toPath() );
    }
    catch ( IOException ioe )
    {
        log.warn( String.format( "Temporary file '%s' could not be deleted.", file.toPath() ), ioe );
    }

    log.debug( String.format( "File resource saved with key: %s", fileResource.getStorageKey() ) );

    return fileResource.getStorageKey();
}
 

@Override
public void copyContent( String key, OutputStream output )
    throws IOException, NoSuchElementException
{
    if ( !blobExists( key ) )
    {
        throw new NoSuchElementException( "key '" + key + "' not found." );
    }

    Blob blob = getBlob( key );

    try ( InputStream in = blob.getPayload().openStream() )
    {
        IOUtils.copy( in, output );
    }
}
 

public static ReadHandle open(ScheduledExecutorService executor,
                              BlobStore blobStore, String bucket, String key, String indexKey,
                              VersionCheck versionCheck,
                              long ledgerId, int readBufferSize)
        throws IOException {
    Blob blob = blobStore.getBlob(bucket, indexKey);
    versionCheck.check(indexKey, blob);
    OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create();
    OffloadIndexBlock index = indexBuilder.fromStream(blob.getPayload().openStream());

    BackedInputStream inputStream = new BlobStoreBackedInputStreamImpl(blobStore, bucket, key,
        versionCheck,
        index.getDataObjectLength(),
        readBufferSize);
    return new BlobStoreBackedReadHandleImpl(ledgerId, index, inputStream, executor);
}
 

@Test
public void testReadingFullObject() throws Exception {
    String objectKey = "testReadingFull";
    int objectSize = 12345;
    RandomInputStream toWrite = new RandomInputStream(0, objectSize);
    RandomInputStream toCompare = new RandomInputStream(0, objectSize);

    Payload payload = Payloads.newInputStreamPayload(toWrite);
    payload.getContentMetadata().setContentLength((long)objectSize);
    Blob blob = blobStore.blobBuilder(objectKey)
        .payload(payload)
        .contentLength((long)objectSize)
        .build();
    String ret = blobStore.putBlob(BUCKET, blob);
    log.debug("put blob: {} in Bucket: {}, in blobStore, result: {}", objectKey, BUCKET, ret);

    BackedInputStream toTest = new BlobStoreBackedInputStreamImpl(blobStore, BUCKET, objectKey,
                                                             (key, md) -> {},
                                                             objectSize, 1000);
    assertStreamsMatch(toTest, toCompare);
}
 

@Test
public void testReadingFullObjectByBytes() throws Exception {
    String objectKey = "testReadingFull2";
    int objectSize = 12345;
    RandomInputStream toWrite = new RandomInputStream(0, objectSize);
    RandomInputStream toCompare = new RandomInputStream(0, objectSize);

    Payload payload = Payloads.newInputStreamPayload(toWrite);
    payload.getContentMetadata().setContentLength((long)objectSize);
    Blob blob = blobStore.blobBuilder(objectKey)
        .payload(payload)
        .contentLength((long)objectSize)
        .build();
    String ret = blobStore.putBlob(BUCKET, blob);
    log.debug("put blob: {} in Bucket: {}, in blobStore, result: {}", objectKey, BUCKET, ret);

    BackedInputStream toTest = new BlobStoreBackedInputStreamImpl(blobStore, BUCKET, objectKey,
                                                             (key, md) -> {},
                                                             objectSize, 1000);
    assertStreamsMatchByBytes(toTest, toCompare);
}
 

private static void validateBlob(Blob blob) throws IOException {
    assertThat(blob).isNotNull();

    ContentMetadata contentMetadata =
            blob.getMetadata().getContentMetadata();
    assertThat(contentMetadata.getContentDisposition())
            .isEqualTo("attachment; filename=foo.mp4");
    assertThat(contentMetadata.getContentEncoding())
            .isEqualTo("compress");
    assertThat(contentMetadata.getContentLength())
            .isEqualTo(BYTE_SOURCE.size());
    assertThat(contentMetadata.getContentType())
            .isEqualTo(MediaType.MP4_AUDIO.toString());

    assertThat(blob.getMetadata().getUserMetadata())
            .isEqualTo(ImmutableMap.of("key", "value"));

    try (InputStream actual = blob.getPayload().openStream();
            InputStream expected = BYTE_SOURCE.openStream()) {
        assertThat(actual).hasContentEqualTo(expected);
    }
}
 

@Override
public Reader get( EntityReference entityReference )
    throws EntityStoreException
{
    Blob blob = storeContext.getBlobStore().getBlob( container, entityReference.identity().toString() );
    if( blob == null )
    {
        throw new EntityNotFoundException( entityReference );
    }
    Payload payload = blob.getPayload();
    if( payload == null )
    {
        throw new EntityNotFoundException( entityReference );
    }
    try( InputStream input = payload.openStream() )
    {
        String state = new Scanner( input, StandardCharsets.UTF_8.name() ).useDelimiter( "\\Z" ).next();
        return new StringReader( state );
    }
    catch( IOException ex )
    {
        throw new EntityStoreException( "Unable to read entity state for: " + entityReference, ex );
    }
}
 

@Override
public String putBlob(String containerName, Blob blob,
        PutOptions options) {
    long length;
    try (InputStream is = blob.getPayload().openStream()) {
        length = ByteStreams.copy(is, ByteStreams.nullOutputStream());
    } catch (IOException ioe) {
        throw new RuntimeException(ioe);
    }

    byte[] array = Longs.toByteArray(length);
    ByteSourcePayload payload = new ByteSourcePayload(
            ByteSource.wrap(array));
    payload.setContentMetadata(blob.getPayload().getContentMetadata());
    payload.getContentMetadata().setContentLength((long) array.length);
    payload.getContentMetadata().setContentMD5((HashCode) null);
    blob.setPayload(payload);

    return super.putBlob(containerName, blob, options);
}
 

@Override
@Nullable
public Blob getBlob(String container, String name, GetOptions options) {
    Blob blob = super.getBlob(container, name, options);
    if (blob == null) {
        return null;
    }

    byte[] array;
    try (InputStream is = blob.getPayload().openStream()) {
        array = ByteStreams.toByteArray(is);
    } catch (IOException ioe) {
        throw new RuntimeException(ioe);
    }

    long length = Longs.fromByteArray(array);
    ByteSourcePayload payload = new ByteSourcePayload(
            new NullByteSource().slice(0, length));
    payload.setContentMetadata(blob.getPayload().getContentMetadata());
    payload.getContentMetadata().setContentLength(length);
    payload.getContentMetadata().setContentMD5((HashCode) null);
    blob.setPayload(payload);
    blob.getMetadata().setSize(length);
    return blob;
}
 

public int readAfterDelete() throws IOException {
    int count = 0;
    for (int i = 0; i < iterations; ++i) {
        String blobName = makeBlobName();
        blobStoreRead.putBlob(containerName, makeBlob(blobName, payload1));
        blobStore.removeBlob(containerName, blobName);
        Blob getBlob = blobStoreRead.getBlob(containerName, blobName);
        if (getBlob != null) {
            ++count;
            try (Payload payload = getBlob.getPayload();
                 InputStream is = payload.openStream()) {
                ByteStreams.copy(is, ByteStreams.nullOutputStream());
            }
        }
    }
    return count;
}
 

@Override
public String putBlob(final String containerName, Blob blob,
        final PutOptions options) {
    final String nearName = blob.getMetadata().getName();
    String nearETag = writeStore.putBlob(containerName, blob, options);
    schedule(new Callable<String>() {
            @Override
            public String call() {
                Blob nearBlob = writeStore.getBlob(containerName, nearName);
                String farETag = delegate().putBlob(containerName,
                        nearBlob, options);
                return farETag;
            }
        });
    return nearETag;
}
 

@Test
public void testCreateBlobGetBlob() throws Exception {
    String blobName = createRandomBlobName();
    Blob blob = makeBlob(nullBlobStore, blobName);
    nullBlobStore.putBlob(containerName, blob);

    blob = nullBlobStore.getBlob(containerName, blobName);
    validateBlobMetadata(blob.getMetadata());

    // content differs, only compare length
    try (InputStream actual = blob.getPayload().openStream();
            InputStream expected = BYTE_SOURCE.openStream()) {
        long actualLength = ByteStreams.copy(actual,
                ByteStreams.nullOutputStream());
        long expectedLength = ByteStreams.copy(expected,
                ByteStreams.nullOutputStream());
        assertThat(actualLength).isEqualTo(expectedLength);
    }

    PageSet<? extends StorageMetadata> pageSet = nullBlobStore.list(
            containerName);
    assertThat(pageSet).hasSize(1);
    StorageMetadata sm = pageSet.iterator().next();
    assertThat(sm.getName()).isEqualTo(blobName);
    assertThat(sm.getSize()).isEqualTo(0);
}
 

/**
 * Upload an object from a String with metadata using the BlobStore API.
 */
private void uploadObjectFromStringWithMetadata() {
   System.out.format("Upload Object From String With Metadata%n");

   String filename = "uploadObjectFromStringWithMetadata.txt";

   Map<String, String> userMetadata = new HashMap<String, String>();
   userMetadata.put("key1", "value1");

   ByteSource source = ByteSource.wrap("uploadObjectFromString".getBytes());

   Blob blob = blobStore.blobBuilder(filename)
         .payload(Payloads.newByteSourcePayload(source))
         .userMetadata(userMetadata)
         .build();

   blobStore.putBlob(CONTAINER, blob);

   System.out.format("  %s%n", filename);
}
 

public void deleteBucket() {

        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

        Properties overrides = new Properties();
        overrides.setProperty( "s3" + ".identity", accessId );
        overrides.setProperty( "s3" + ".credential", secretKey );

        Blob bo = null;
        BlobStore blobStore = null;
        final Iterable<? extends Module> MODULES = ImmutableSet
                .of(new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
                        new NettyPayloadModule());

        BlobStoreContext context =
                ContextBuilder.newBuilder("s3").credentials( accessId, secretKey ).modules( MODULES )
                        .overrides( overrides ).buildView( BlobStoreContext.class );

        blobStore = context.getBlobStore();
        blobStore.deleteContainer( bucketName );
    }
 

@Override
public InputStream read(BucketName bucketName, BlobId blobId) throws ObjectStoreException {
    ObjectStorageBucketName resolvedBucketName = bucketNameResolver.resolve(bucketName);
    Blob blob = blobStore.getBlob(resolvedBucketName.asString(), blobId.asString());

    try {
        if (blob != null) {
            return payloadCodec.read(new Payload(blob.getPayload(), Optional.empty()));
        } else {
            throw new ObjectNotFoundException("fail to load blob with id " + blobId);
        }
    } catch (IOException cause) {
        throw new ObjectStoreException(
            "Failed to readBytes blob " + blobId.asString(),
            cause);
    }
}
 

@Override
public Mono<BlobId> save(BucketName bucketName, byte[] data, StoragePolicy storagePolicy) {
    Preconditions.checkNotNull(data);
    ObjectStorageBucketName resolvedBucketName = bucketNameResolver.resolve(bucketName);

    return Mono.fromCallable(() -> blobIdFactory.forPayload(data))
        .flatMap(blobId -> {
            Payload payload = payloadCodec.write(data);

            Blob blob = blobStore.blobBuilder(blobId.asString())
                .payload(payload.getPayload())
                .contentLength(payload.getLength().orElse(Long.valueOf(data.length)))
                .build();

            return blobPutter.putDirectly(resolvedBucketName, blob)
                .thenReturn(blobId);
        });
}
 

/**
 * {@inheritDoc}
 */
@Override
public InputStream getInputStream(String id, String root, String filePath) throws IOException {
    ContainerAndName can = getContainerAndName(id, root, filePath);
    Blob blob = getBlobStore().getBlob(can.container, can.name);
    if (blob == null){
        throw new IOException("No object found for " + id);
    }

    StorageMetadata metadata = blob.getMetadata();
    Long size = metadata.getSize();

    if (size != null && size.longValue() > maxBlobStreamSize) {
        return streamFromTempFile(blob, size);
    } else {
        // SAK-30325: why can't we just send the stream straight back: blob.getPayload().openStream() ?
        // Good question, but it doesn't work properly unless the stream is fully copied and re-streamed....
        return new ByteArrayInputStream(FileCopyUtils.copyToByteArray(blob.getPayload().openStream()));
    }
}
 

@Override
public byte[] getBytes() {
    try {
        Blob blob = blobStore.getBlob(containerName, blobName);
        if (blob==null) return null;
        return Streams.readFullyAndClose(blob.getPayload().openStream());
    } catch (IOException e) {
        Exceptions.propagateIfFatal(e);
        throw new IllegalStateException("Error reading blobstore "+containerName+" "+blobName+": "+e, e);
    }
}
 

private void put(String name, String content) {
    BlobStore blobStore = context.getBlobStore();
    byte[] bytes = content.getBytes(Charsets.UTF_8);
    Blob blob = blobStore.blobBuilder(name)
            .payload(ByteSource.wrap(bytes))
            .contentLength(bytes.length)
            .build();
    try {
        blobStore.putBlob(containerName, blob);
    } catch (Exception e) {
        LOG.error("PUT " + name + " failed", e);
    }
}
 

private void putBlobWithRetries(Blob blob, int retries) {
    for (int i = 1; i <= retries; i++) {
        try {
            blobStore.putBlob(container, blob, PutOptions.Builder.multipart());
            return;
        } catch (HttpResponseException e) {
            LOGGER.warn(MessageFormat.format(Messages.ATTEMPT_TO_UPLOAD_BLOB_FAILED, i, retries, e.getMessage()), e);
            if (i == retries) {
                throw e;
            }
        }
        MiscUtil.sleep(i * getRetryWaitTime());
    }
}
 

@Test
public void testCreateBlobBlobMetadata() throws Exception {
    String blobName = createRandomBlobName();
    Blob blob = makeBlob(nullBlobStore, blobName);
    nullBlobStore.putBlob(containerName, blob);
    BlobMetadata metadata = nullBlobStore.blobMetadata(containerName,
            blobName);
    validateBlobMetadata(metadata);
}
 

private Mono<BlobId> saveBigStream(BucketName bucketName, InputStream data) {
    ObjectStorageBucketName resolvedBucketName = bucketNameResolver.resolve(bucketName);

    return Mono.fromCallable(blobIdFactory::randomId)
        .flatMap(tmpId -> {
            HashingInputStream hashingInputStream = new HashingInputStream(Hashing.sha256(), data);
            Payload payload = payloadCodec.write(hashingInputStream);
            Blob blob = blobStore.blobBuilder(tmpId.asString())
                .payload(payload.getPayload())
                .build();

            Supplier<BlobId> blobIdSupplier = () -> blobIdFactory.from(hashingInputStream.hash().toString());
            return blobPutter.putAndComputeId(resolvedBucketName, blob, blobIdSupplier);
        });
}
 

public void put(ByteSource payload) {
        blobStore.createContainerInLocation(null, containerName);
        // seems not needed, at least not w SoftLayer
//        blobStore.createDirectory(containerName, directoryName);
        Blob blob;
        try {
            blob = blobStore.blobBuilder(blobName).payload(payload)
                    .contentLength(payload.size())
                    .build();
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }
        blobStore.putBlob(containerName, blob);
    }