com.google.cloud.storage.Blob Java Examples

The following examples show how to use com.google.cloud.storage.Blob. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: GCSErrorWriter.java    From beast with Apache License 2.0 7 votes vote down vote up
private void storeMessagesInGCS(final List<Record> errorRecords) throws StorageException {
    //get all messages to serialize per topic
    final Map<String, GCSInvalidMessagesWrapper> topicMessagesMap = getMessagesToSerializePerTopic(errorRecords);
    //serialize the messages in GCS for each topic - a file with all messages per topic is stored in GCS
    topicMessagesMap.keySet().forEach(topicName -> {
        final String fileName = UUID.randomUUID().toString();
        final String pathPrefix = gcsBasePathPrefix + "/" + topicName + "/" + getFormattedDatePrefix(Instant.now()) + "/";
        final BlobId blobId = BlobId.of(gcsBucket, pathPrefix + fileName);
        final Map<String, String> metaDataMap = new HashMap<>();
        metaDataMap.put("topic", topicName);
        metaDataMap.put("uuid", fileName);
        final BlobInfo objectInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").setMetadata(metaDataMap).build();
        try {
            final Blob objectCreated = gcsStore.create(objectInfo, topicMessagesMap.get(topicName).getBytes());
        } catch (JsonProcessingException jpe) {
            log.error("Exception::Failed to write to GCS: {} records size: {}", jpe, errorRecords.size());
            throw new BQErrorHandlerException(jpe.getMessage());
        }
    });
    log.info("Pushing {} records to GCS success?: {}", errorRecords.size(), true);
}
 
Example #2
Source File: GCPRestorer.java    From cassandra-backup with Apache License 2.0 6 votes vote down vote up
@Override
public Path downloadFileToDir(final Path destinationDir, final Path remotePrefix, final Predicate<String> keyFilter) throws Exception {

    final Page<Blob> blobs = list(request.storageLocation.bucket, remotePrefix.toString());
    final List<Blob> blobItems = new ArrayList<>();

    for (final Blob blob : blobs.iterateAll()) {
        if (keyFilter.test(blob.getName())) {
            blobItems.add(blob);
        }
    }

    if (blobItems.size() != 1) {
        throw new IllegalStateException(format("There is not one key which satisfies key filter: %s", blobItems.toString()));
    }

    final String blobItemPath = blobItems.get(0).getName();
    final String fileName = blobItemPath.split("/")[blobItemPath.split("/").length - 1];

    final Path destination = destinationDir.resolve(fileName);

    downloadFile(destination, objectKeyToRemoteReference(remotePrefix.resolve(fileName)));

    return destination;
}
 
Example #3
Source File: DeploymentFailsForFirestoreNativeIT.java    From nexus-blobstore-google-cloud with Eclipse Public License 1.0 6 votes vote down vote up
@After
public void destroyBucket() throws IOException {
  Storage storage = StorageOptions.newBuilder()
      .setCredentials(ServiceAccountCredentials.fromStream(new FileInputStream(firestoreNativeConfiguration)))
      .build().getService();
  log.debug("Deleting files from " + bucketName);
  // must delete all the files within the bucket before we can delete the bucket
  Iterator<Blob> list = storage.list(bucketName,
      Storage.BlobListOption.prefix("")).iterateAll()
      .iterator();
  list.forEachRemaining(blob -> blob.delete());

  storage.delete(bucketName);

  log.info(bucketName + "bucket deleted");
}
 
Example #4
Source File: GCPBucketService.java    From cassandra-backup with Apache License 2.0 6 votes vote down vote up
@Override
public void delete(final String bucketName) {
    final Bucket bucket = storage.get(bucketName);

    if (bucket != null) {
        for (final Blob blob : bucket.list().iterateAll()) {
            if (!blob.delete()) {
                throw new GCPModuleException(format("Blob %s was not deleted!", blob.getName()));
            }
        }

        if (!storage.delete(bucket.getName())) {
            throw new GCPModuleException(format("GCP bucket %s was not deleted!", bucket.getName()));
        }
    }
}
 
Example #5
Source File: StorageSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example of how to release a temporary hold for a blob */
public Blob releaseTemporaryHold(String bucketName, String blobName) throws StorageException {
  // [START storage_release_temporary_hold]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  // The name of a blob, e.g. "my-blob"
  // String blobName = "my-blob";

  BlobId blobId = BlobId.of(bucketName, blobName);
  Blob blob = storage.update(BlobInfo.newBuilder(blobId).setTemporaryHold(false).build());

  System.out.println("Temporary hold was released for " + blobName);
  // [END storage_release_temporary_hold]
  return blob;
}
 
Example #6
Source File: ITBucketSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
@Test
public void testRequesterPays() throws Exception {
  EnableRequesterPays.enableRequesterPays(PROJECT_ID, BUCKET);
  Bucket bucket = storage.get(BUCKET);
  assertTrue(bucket.requesterPays());
  String projectId = ServiceOptions.getDefaultProjectId();
  String blobName = "test-create-empty-blob-requester-pays";
  byte[] content = {0xD, 0xE, 0xA, 0xD};
  Blob remoteBlob =
      bucket.create(blobName, content, Bucket.BlobTargetOption.userProject(projectId));
  assertNotNull(remoteBlob);
  DownloadRequesterPaysObject.downloadRequesterPaysObject(
      projectId, BUCKET, blobName, Paths.get(blobName));
  byte[] readBytes = Files.readAllBytes(Paths.get(blobName));
  assertArrayEquals(content, readBytes);
  DisableRequesterPays.disableRequesterPays(PROJECT_ID, BUCKET);
  assertFalse(storage.get(BUCKET).requesterPays());
}
 
Example #7
Source File: DocumentOcrTemplate.java    From spring-cloud-gcp with Apache License 2.0 6 votes vote down vote up
/**
 * Parses the OCR output files who have the specified {@code jsonFilesetPrefix}. This
 * method assumes that all of the OCR output files with the prefix are a part of the same
 * document.
 *
 * @param jsonOutputFilePathPrefix the folder location containing all of the JSON files of
 *     OCR output
 * @return A {@link DocumentOcrResultSet} describing the OCR content of a document
 */
public DocumentOcrResultSet readOcrOutputFileSet(GoogleStorageLocation jsonOutputFilePathPrefix) {
	String nonNullPrefix = (jsonOutputFilePathPrefix.getBlobName() == null)
			? ""
			: jsonOutputFilePathPrefix.getBlobName();

	Page<Blob> blobsInFolder = this.storage.list(
			jsonOutputFilePathPrefix.getBucketName(),
			BlobListOption.currentDirectory(),
			BlobListOption.prefix(nonNullPrefix));

	List<Blob> blobPages =
			StreamSupport.stream(blobsInFolder.getValues().spliterator(), false)
					.filter(blob -> blob.getContentType().equals("application/octet-stream"))
					.collect(Collectors.toList());

	return new DocumentOcrResultSet(blobPages);
}
 
Example #8
Source File: StorageSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example of how to set a temporary hold for a blob */
public Blob setTemporaryHold(String bucketName, String blobName) throws StorageException {
  // [START storage_set_temporary_hold]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  // The name of a blob, e.g. "my-blob"
  // String blobName = "my-blob";

  BlobId blobId = BlobId.of(bucketName, blobName);
  Blob blob = storage.update(BlobInfo.newBuilder(blobId).setTemporaryHold(true).build());

  System.out.println("Temporary hold was set for " + blobName);
  // [END storage_set_temporary_hold]
  return blob;
}
 
Example #9
Source File: StorageExample.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/**
 * Sets the ACL according to the provided {@code params}, using the {@code storage} service. If
 * {@code params.x()} returns a complete blob identity, the {@code params.y()} ACL is added to
 * the blob. If {@code params.x().name()} is empty, the {@code params.y()} ACL is added to the
 * bucket identified by {@code params.x().bucket()}.
 */
@Override
public void run(Storage storage, Tuple<BlobId, Acl> params) {
  BlobId blobId = params.x();
  Acl acl = params.y();
  if (blobId.getName().isEmpty()) {
    Bucket bucket = storage.get(blobId.getBucket());
    if (bucket == null) {
      System.out.printf("Bucket %s does not exist%n", blobId.getBucket());
      return;
    }
    acl = bucket.createAcl(acl);
    System.out.printf("Added ACL %s to bucket %s%n", acl, blobId.getBucket());
  } else {
    Blob blob = storage.get(blobId);
    if (blob == null) {
      System.out.printf("Blob %s does not exist%n", blobId);
      return;
    }
    acl = blob.createAcl(acl);
    System.out.printf("Added ACL %s to blob %s%n", acl, blobId);
  }
}
 
Example #10
Source File: StorageSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example of updating information on several blobs using a single batch request. */
// [TARGET update(Iterable)]
// [VARIABLE "my_unique_bucket"]
// [VARIABLE "my_blob_name1"]
// [VARIABLE "my_blob_name2"]
public List<Blob> batchUpdateIterable(String bucketName, String blobName1, String blobName2) {
  // [START batchUpdateIterable]
  Blob firstBlob = storage.get(bucketName, blobName1);
  Blob secondBlob = storage.get(bucketName, blobName2);
  List<BlobInfo> blobs = new LinkedList<>();
  blobs.add(firstBlob.toBuilder().setContentType("text/plain").build());
  blobs.add(secondBlob.toBuilder().setContentType("text/plain").build());
  List<Blob> updatedBlobs = storage.update(blobs);
  // [END batchUpdateIterable]
  return updatedBlobs;
}
 
Example #11
Source File: GcsPinotFS.java    From incubator-pinot with Apache License 2.0 6 votes vote down vote up
/**
 * Determines if a path is a directory that is not empty
 * @param uri The path under the gcs bucket
 * @return {@code true} if the path is a non-empty directory,
 *         {@code false} otherwise
 */
private boolean isEmptyDirectory(URI uri) throws IOException {
  if (!isDirectory(uri)) {
    return false;
  }
  String prefix = normalizeToDirectoryPrefix(uri);
  boolean isEmpty = true;
  Page<Blob> page;
  if (prefix.equals(DELIMITER)) {
    page = getBucket(uri).list();
  } else {
    page = getBucket(uri).list(Storage.BlobListOption.prefix(prefix));
  }
  for (Blob blob : page.iterateAll()) {
    if (blob.getName().equals(prefix)) {
      continue;
    } else {
      isEmpty = false;
      break;
    }
  }
  return isEmpty;
}
 
Example #12
Source File: BigQueryLoadTest.java    From gcp-ingestion with Mozilla Public License 2.0 6 votes vote down vote up
private Optional<BigQuery.BigQueryErrors> load() {
  final Blob blob = mock(Blob.class);
  final String name = "OUTPUT_TABLE=dataset.table/x";
  when(blob.getName()).thenReturn(name);
  when(blob.getBlobId()).thenReturn(BlobId.of("bucket", name));
  when(blob.getSize()).thenReturn(1L);
  try {
    new BigQuery.Load(bigQuery, storage, 0, 0, Duration.ZERO, ForkJoinPool.commonPool(),
        BigQuery.Load.Delete.onSuccess).apply(blob).join();
  } catch (CompletionException e) {
    if (e.getCause() instanceof BigQuery.BigQueryErrors) {
      return Optional.of((BigQuery.BigQueryErrors) e.getCause());
    }
  }
  return Optional.empty();
}
 
Example #13
Source File: BigQueryHistoricalRetriever.java    From feast with Apache License 2.0 6 votes vote down vote up
private List<String> parseOutputFileURIs(String feastJobId) {
  String scheme = jobStagingLocation().substring(0, jobStagingLocation().indexOf("://"));
  String stagingLocationNoScheme =
      jobStagingLocation().substring(jobStagingLocation().indexOf("://") + 3);
  String bucket = stagingLocationNoScheme.split("/")[0];
  List<String> prefixParts = new ArrayList<>();
  prefixParts.add(
      stagingLocationNoScheme.contains("/") && !stagingLocationNoScheme.endsWith("/")
          ? stagingLocationNoScheme.substring(stagingLocationNoScheme.indexOf("/") + 1)
          : "");
  prefixParts.add(feastJobId);
  String prefix = String.join("/", prefixParts) + "/";

  List<String> fileUris = new ArrayList<>();
  for (Blob blob : storage().list(bucket, Storage.BlobListOption.prefix(prefix)).iterateAll()) {
    fileUris.add(String.format("%s://%s/%s", scheme, blob.getBucket(), blob.getName()));
  }
  return fileUris;
}
 
Example #14
Source File: GcsRepository.java    From hawkbit-extensions with Eclipse Public License 1.0 6 votes vote down vote up
@Override
public AbstractDbArtifact getArtifactBySha1(final String tenant, final String sha1Hash) {
    final String key = objectKey(tenant, sha1Hash);

    LOG.info("Retrieving GCS object from bucket {} and key {}", gcsProperties.getBucketName(), key);
    final Blob blob = gcsStorage.get(gcsProperties.getBucketName(), key);
    if (blob == null || !blob.exists()) {
        return null;
    }
    // the MD5Content is stored in the ETag
    return new GcsArtifact(gcsStorage, gcsProperties, key, sha1Hash,
            new DbArtifactHash(sha1Hash,
                    BaseEncoding.base16().lowerCase().encode(BaseEncoding.base64().decode(blob.getMd5())), null),
            blob.getSize(), blob.getContentType());

}
 
Example #15
Source File: ListGCSBucketTest.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testOldValues() throws Exception {
    reset(storage, mockBlobPage);
    final ListGCSBucket processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    final Iterable<Blob> mockList = ImmutableList.of(
            buildMockBlob("blob-bucket-1", "blob-key-1", 2L)
    );

    when(mockBlobPage.getValues()).thenReturn(mockList);
    when(mockBlobPage.getNextPage()).thenReturn(null);
    when(storage.list(anyString(), any(Storage.BlobListOption[].class))).thenReturn(mockBlobPage);

    runner.enqueue("test");
    runner.enqueue("test2");
    runner.run(2);

    runner.assertAllFlowFilesTransferred(ListGCSBucket.REL_SUCCESS);
    runner.assertTransferCount(ListGCSBucket.REL_SUCCESS, 1);

    assertEquals("blob-key-1", runner.getStateManager().getState(Scope.CLUSTER).get(ListGCSBucket.CURRENT_KEY_PREFIX+"0"));
    assertEquals("2", runner.getStateManager().getState(Scope.CLUSTER).get(ListGCSBucket.CURRENT_TIMESTAMP));
}
 
Example #16
Source File: StreamingAnnotationToStorageIT.java    From java-docs-samples with Apache License 2.0 6 votes vote down vote up
@Test
public void testStreamingAnnotationToStorage() {
  String gcsUri = String.format("gs://%s/%s", PROJECT_ID, OUTPUT_PREFIX);
  StreamingAnnotationToStorage.streamingAnnotationToStorage("resources/cat.mp4", gcsUri);
  String got = bout.toString();

  assertThat(got).contains(String.format("Storage Uri: %s", gcsUri));

  Storage storage = StorageOptions.getDefaultInstance().getService();

  Page<Blob> blobs =
      storage.list(
          PROJECT_ID,
          BlobListOption.currentDirectory(),
          BlobListOption.prefix(OUTPUT_PREFIX + "/"));

  deleteDirectory(storage, blobs);
}
 
Example #17
Source File: GCSStorage.java    From digdag with Apache License 2.0 6 votes vote down vote up
@Override
public void list(String objectPrefix, FileListing callback)
{
    checkArgument(objectPrefix != null, "objectPrefix is null");

    String errorMessage = "listing files on bucket " + bucket + " prefix " + objectPrefix;
    Page<Blob> blobs = getWithRetry(errorMessage, () ->
            storage.list(bucket, BlobListOption.prefix(objectPrefix))
    );

    List<StorageObjectSummary> objectSummaryList = new ArrayList<>();
    for (Blob blob : blobs.iterateAll()) {
        objectSummaryList.add(
                StorageObjectSummary.builder()
                        .key(blob.getName())
                        .contentLength(blob.getSize())
                        .lastModified(convertToInstant(blob))
                        .build()
        );
    }
    callback.accept(objectSummaryList);
}
 
Example #18
Source File: DownloadRequesterPaysObject.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
public static void downloadRequesterPaysObject(
    String projectId, String bucketName, String objectName, Path destFilePath) {
  // The project ID to bill
  // String projectId = "my-billable-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // The ID of your GCS object
  // String objectName = "your-object-name";

  // The path to which the file should be downloaded
  // Path destFilePath = Paths.get("/local/path/to/file.txt");

  Storage storage = StorageOptions.getDefaultInstance().getService();
  Blob blob = storage.get(BlobId.of(bucketName, objectName));
  blob.downloadTo(destFilePath, Blob.BlobSourceOption.userProject(projectId));

  System.out.println(
      "Object " + objectName + " downloaded to " + destFilePath + " and billed to " + projectId);
}
 
Example #19
Source File: StorageSnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
/** Example of creating a blob from a byte array. */
// [TARGET create(BlobInfo, byte[], BlobTargetOption...)]
// [VARIABLE "my_unique_bucket"]
// [VARIABLE "my_blob_name"]
public Blob createBlobFromByteArray(String bucketName, String blobName) {
  // [START createBlobFromByteArray]
  BlobId blobId = BlobId.of(bucketName, blobName);
  BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
  Blob blob = storage.create(blobInfo, "Hello, World!".getBytes(UTF_8));
  // [END createBlobFromByteArray]
  return blob;
}
 
Example #20
Source File: ITStorageSnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
@Test
public void testBucketRetention() {
  Bucket bucket = storageSnippets.setRetentionPolicy(BUCKET, RETENTION_PERIOD);
  assertEquals(bucket.getRetentionPeriod(), RETENTION_PERIOD);
  assertNotNull(bucket.getRetentionEffectiveTime());
  bucket = storageSnippets.getRetentionPolicy(BUCKET);
  assertEquals(bucket.getRetentionPeriod(), RETENTION_PERIOD);
  assertNotNull(bucket.getRetentionEffectiveTime());
  assertNull(bucket.retentionPolicyIsLocked());
  bucket = storageSnippets.enableDefaultEventBasedHold(BUCKET);
  assertTrue(bucket.getDefaultEventBasedHold());
  bucket = storageSnippets.getDefaultEventBasedHold(BUCKET);
  assertTrue(bucket.getDefaultEventBasedHold());
  String blobName = "test-create-empty-blob-retention-policy";
  Blob remoteBlob = bucket.create(blobName, BLOB_BYTE_CONTENT);
  assertTrue(remoteBlob.getEventBasedHold());
  remoteBlob = storageSnippets.setEventBasedHold(BUCKET, blobName);
  assertTrue(remoteBlob.getEventBasedHold());
  remoteBlob = storageSnippets.releaseEventBasedHold(BUCKET, blobName);
  assertFalse(remoteBlob.getEventBasedHold());
  assertNotNull(remoteBlob.getRetentionExpirationTime());
  bucket = storageSnippets.removeRetentionPolicy(BUCKET);
  assertNull(bucket.getRetentionPeriod());
  assertNull(bucket.getRetentionEffectiveTime());
  bucket = storageSnippets.disableDefaultEventBasedHold(BUCKET);
  assertFalse(bucket.getDefaultEventBasedHold());
  remoteBlob = storageSnippets.setTemporaryHold(BUCKET, blobName);
  assertTrue(remoteBlob.getTemporaryHold());
  remoteBlob = storageSnippets.releaseTemporaryHold(BUCKET, blobName);
  assertFalse(remoteBlob.getTemporaryHold());
}
 
Example #21
Source File: StorageSnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
/** Example of getting information on several blobs using a single batch request. */
// [TARGET get(Iterable)]
// [VARIABLE "my_unique_bucket"]
// [VARIABLE "my_blob_name1"]
// [VARIABLE "my_blob_name2"]
public List<Blob> batchGetIterable(String bucketName, String blobName1, String blobName2) {
  // [START batchGetIterable]
  List<BlobId> blobIds = new LinkedList<>();
  blobIds.add(BlobId.of(bucketName, blobName1));
  blobIds.add(BlobId.of(bucketName, blobName2));
  List<Blob> blobs = storage.get(blobIds);
  // [END batchGetIterable]
  return blobs;
}
 
Example #22
Source File: BucketSnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
/** Example of creating a blob in the bucket from an input stream. */
// [TARGET create(String, InputStream, BlobWriteOption...)]
// [VARIABLE "my_blob_name"]
public Blob createBlobFromInputStream(String blobName) {
  // [START createBlobFromInputStream]
  InputStream content = new ByteArrayInputStream("Hello, World!".getBytes(UTF_8));
  Blob blob = bucket.create(blobName, content);
  // [END createBlobFromInputStream]
  return blob;
}
 
Example #23
Source File: GoogleStorageTests.java    From spring-cloud-gcp with Apache License 2.0 5 votes vote down vote up
@Test
public void testWritableOutputStream() throws Exception {
	String location = "gs://test-spring/test";
	Blob blob = mock(Blob.class);
	WriteChannel writeChannel = mock(WriteChannel.class);
	when(blob.writer()).thenReturn(writeChannel);
	when(blob.exists()).thenReturn(true);
	when(this.mockStorage.get(BlobId.of("test-spring", "test"))).thenReturn(blob);

	GoogleStorageResource resource = new GoogleStorageResource(this.mockStorage, location);
	OutputStream os = resource.getOutputStream();
	Assert.assertNotNull(os);
}
 
Example #24
Source File: ListGCSBucketTest.java    From nifi with Apache License 2.0 5 votes vote down vote up
private Blob buildMockBlob(String bucket, String key, long updateTime) {
    final Blob blob = mock(Blob.class);
    when(blob.getBucket()).thenReturn(bucket);
    when(blob.getName()).thenReturn(key);
    when(blob.getUpdateTime()).thenReturn(updateTime);
    return blob;
}
 
Example #25
Source File: FetchGCSObjectTest.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testAclOwnerUser() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    final Blob blob = mock(Blob.class);

    final Acl.User mockUser = mock(Acl.User.class);
    when(mockUser.getEmail()).thenReturn(OWNER_USER_EMAIL);
    when(blob.getOwner()).thenReturn(mockUser);

    when(storage.get(any(BlobId.class))).thenReturn(blob);
    when(storage.reader(any(BlobId.class), any(Storage.BlobSourceOption.class))).thenReturn(new MockReadChannel(CONTENT));

    runner.enqueue("");

    runner.run();

    verify(storage).get(any(BlobId.class));
    verify(storage).reader(any(BlobId.class), any(Storage.BlobSourceOption.class));

    runner.assertAllFlowFilesTransferred(FetchGCSObject.REL_SUCCESS);
    runner.assertTransferCount(FetchGCSObject.REL_SUCCESS, 1);
    final MockFlowFile flowFile = runner.getFlowFilesForRelationship(FetchGCSObject.REL_SUCCESS).get(0);

    assertEquals(
            OWNER_USER_EMAIL,
            flowFile.getAttribute(OWNER_ATTR)
    );

    assertEquals(
            "user",
            flowFile.getAttribute(OWNER_TYPE_ATTR)
    );
}
 
Example #26
Source File: SuccessfulDeploymentIT.java    From nexus-blobstore-google-cloud with Eclipse Public License 1.0 5 votes vote down vote up
@After
public void destroyBucket() throws Exception {
  Storage storage = StorageOptions.newBuilder().build().getService();
  log.debug("Deleting files from {}", bucketName);
  // must delete all the files within the bucket before we can delete the bucket
  Iterator<Blob> list = storage.list(bucketName,
      Storage.BlobListOption.prefix("")).iterateAll()
      .iterator();
  list.forEachRemaining(blob -> blob.delete());

  storage.delete(bucketName);

  log.info("{} bucket deleted", bucketName);
}
 
Example #27
Source File: OldGCSNotebookRepo.java    From zeppelin with Apache License 2.0 5 votes vote down vote up
@Override
public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
  try {
    List<OldNoteInfo> infos = new ArrayList<>();
    Iterable<Blob> blobsUnderDir;
    if (basePath.isPresent()) {
      blobsUnderDir = storage
        .list(bucketName, BlobListOption.prefix(this.basePath.get() + "/"))
        .iterateAll();
    } else {
      blobsUnderDir = storage
        .list(bucketName)
        .iterateAll();
    }
    for (Blob b : blobsUnderDir) {
      Matcher matcher = noteNamePattern.matcher(b.getName());
      if (matcher.matches()) {
        // Callers only use the id field, so do not fetch each note
        // This matches the implementation in FileSystemNoteRepo#list
        infos.add(new OldNoteInfo(matcher.group(1), "", null));
      }
    }
    return infos;
  } catch (StorageException se) {
    throw new IOException("Could not list GCS directory: " + se.getMessage(), se);
  }
}
 
Example #28
Source File: StorageSnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
/** Example of creating a blob with no content. */
// [TARGET create(BlobInfo, BlobTargetOption...)]
// [VARIABLE "my_unique_bucket"]
// [VARIABLE "my_blob_name"]
public Blob createBlob(String bucketName, String blobName) {
  // [START createBlob]
  BlobId blobId = BlobId.of(bucketName, blobName);
  BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
  Blob blob = storage.create(blobInfo);
  // [END createBlob]
  return blob;
}
 
Example #29
Source File: GoogleStorageTests.java    From spring-cloud-gcp with Apache License 2.0 5 votes vote down vote up
@Test
public void signedUrlFunctionCalled() throws IOException {
	String location = "gs://test-spring/t1/test.png";
	Storage storage = mock(Storage.class);
	Blob blob = mock(Blob.class);
	when(blob.getBucket()).thenReturn("fakeBucket");
	when(blob.getName()).thenReturn("fakeObject");
	GoogleStorageResource resource = new GoogleStorageResource(storage, location);
	when(storage.get(any(BlobId.class))).thenReturn(blob);
	Storage.SignUrlOption option = Storage.SignUrlOption.httpMethod(HttpMethod.PUT);
	resource.createSignedUrl(TimeUnit.DAYS, 1L, option);
	verify(storage, times(1)).signUrl(any(), eq(1L), eq(TimeUnit.DAYS), eq(option));
}
 
Example #30
Source File: MultipartUploader.java    From nexus-blobstore-google-cloud with Eclipse Public License 1.0 5 votes vote down vote up
/**
 * @param storage an initialized {@link Storage} instance
 * @param bucket the name of the bucket
 * @param destination the the destination (relative to the bucket)
 * @param contents the stream of data to store
 * @return the successfully stored {@link Blob}
 * @throws BlobStoreException if any part of the upload failed
 */
@Override
@Guarded(by = STARTED)
public Blob upload(final Storage storage, final String bucket, final String destination, final InputStream contents) {
  if(isParallel()) {
    return parallelUpload(storage, bucket, destination, contents);
  }
  log.debug("Starting upload for destination {} in bucket {}", destination, bucket);
  BlobInfo blobInfo = BlobInfo.newBuilder(bucket, destination).build();
  Blob result = storage.create(blobInfo, contents, BlobWriteOption.disableGzipContent());
  log.debug("Upload of {} complete", destination);
  return result;
}