com.google.cloud.storage.Storage Java Examples

private static final void deleteObjects() {
  Storage storage = StorageOptions.getDefaultInstance().getService();
  for (BlobInfo info :
      storage
          .list(
              bucketName,
              BlobListOption.versions(true),
              BlobListOption.currentDirectory(),
              BlobListOption.prefix(path + "/"))
          .getValues()) {
    storage.delete(info.getBlobId());
  }
}
 

public static void enableLifecycleManagement(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Bucket bucket = storage.get(bucketName);

  // See the LifecycleRule documentation for additional info on what you can do with lifecycle
  // management rules. This one deletes objects that are over 100 days old.
  // https://googleapis.dev/java/google-cloud-clients/latest/com/google/cloud/storage/BucketInfo.LifecycleRule.html
  bucket
      .toBuilder()
      .setLifecycleRules(
          ImmutableList.of(
              new LifecycleRule(
                  LifecycleRule.LifecycleAction.newDeleteAction(),
                  LifecycleRule.LifecycleCondition.newBuilder().setAge(100).build())))
      .build()
      .update();

  System.out.println("Lifecycle management was enabled and configured for bucket " + bucketName);
}
 

/** Example of setting a default KMS key on a bucket. */
public Bucket setDefaultKmsKey(String bucketName, String kmsKeyName) throws StorageException {
  // [START storage_set_bucket_default_kms_key]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of the existing bucket to set a default KMS key for, e.g. "my-bucket"
  // String bucketName = "my-bucket"

  // The name of the KMS-key to use as a default
  // Key names are provided in the following format:
  // 'projects/<PROJECT>/locations/<LOCATION>/keyRings/<RING_NAME>/cryptoKeys/<KEY_NAME>'
  // String kmsKeyName = ""

  BucketInfo bucketInfo =
      BucketInfo.newBuilder(bucketName).setDefaultKmsKeyName(kmsKeyName).build();

  Bucket bucket = storage.update(bucketInfo);

  System.out.println("Default KMS Key Name: " + bucket.getDefaultKmsKeyName());
  // [END storage_set_bucket_default_kms_key]
  return bucket;
}
 

@Test
public void testAclAttributeProject() throws Exception {
    reset(storage, blob);
    final PutGCSObject processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    when(storage.create(any(BlobInfo.class), any(InputStream.class), any(Storage.BlobWriteOption.class)))
            .thenReturn(blob);

    final Acl.Project mockProject = mock(Acl.Project.class);
    when(mockProject.getProjectId()).thenReturn(OWNER_PROJECT_ID);
    when(blob.getOwner()).thenReturn(mockProject);

    runner.enqueue("test");
    runner.run();

    runner.assertAllFlowFilesTransferred(PutGCSObject.REL_SUCCESS);
    runner.assertTransferCount(PutGCSObject.REL_SUCCESS, 1);

    final MockFlowFile mockFlowFile = runner.getFlowFilesForRelationship(PutGCSObject.REL_SUCCESS).get(0);
    mockFlowFile.assertAttributeEquals(OWNER_ATTR, OWNER_PROJECT_ID);
    mockFlowFile.assertAttributeEquals(OWNER_TYPE_ATTR, "project");
}
 

private void run(Storage storage, Path uploadFrom, BlobInfo blobInfo) throws IOException {
  if (Files.size(uploadFrom) > 1_000_000) {
    // When content is not available or large (1MB or more) it is recommended
    // to write it in chunks via the blob's channel writer.
    try (WriteChannel writer = storage.writer(blobInfo)) {
      byte[] buffer = new byte[1024];
      try (InputStream input = Files.newInputStream(uploadFrom)) {
        int limit;
        while ((limit = input.read(buffer)) >= 0) {
          try {
            writer.write(ByteBuffer.wrap(buffer, 0, limit));
          } catch (Exception ex) {
            ex.printStackTrace();
          }
        }
      }
    }
  } else {
    byte[] bytes = Files.readAllBytes(uploadFrom);
    // create the blob in one request.
    storage.create(blobInfo, bytes);
  }
  System.out.println("Blob was created");
}
 

@Bean
public Storage gcsClient() {
	Storage gcs = mock(Storage.class);

	willAnswer((invocationOnMock) ->
		new PageImpl<>(null, null,
				Stream.of(
						createBlob("gcsbucket", "gamma"),
						createBlob("gcsbucket", "beta"),
						createBlob("gcsbucket", "alpha/alpha"))
						.collect(Collectors.toList())))
			.given(gcs).list(eq("gcsbucket"));

	willAnswer((invocationOnMock) -> mock(ReadChannel.class))
			.given(gcs).reader(eq("gcsbucket"), eq("alpha/alpha"));
	willAnswer((invocationOnMock) -> mock(ReadChannel.class))
			.given(gcs).reader(eq("gcsbucket"), eq("beta"));
	willAnswer((invocationOnMock) -> mock(ReadChannel.class))
			.given(gcs).reader(eq("gcsbucket"), eq("gamma"));

	return gcs;
}
 

/** Example of how to set event-based hold for a blob */
public Blob setEventBasedHold(String bucketName, String blobName) throws StorageException {
  // [START storage_set_event_based_hold]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  // The name of a blob, e.g. "my-blob"
  // String blobName = "my-blob";

  BlobId blobId = BlobId.of(bucketName, blobName);
  Blob blob = storage.update(BlobInfo.newBuilder(blobId).setEventBasedHold(true).build());

  System.out.println("Event-based hold was set for " + blobName);
  // [END storage_set_event_based_hold]
  return blob;
}
 

@Test
public void testYieldOnBadStateRestore() throws Exception {
    reset(storage, mockBlobPage);
    final ListGCSBucket processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    final Iterable<Blob> mockList = ImmutableList.of();

    when(mockBlobPage.getValues()).thenReturn(mockList);
    when(mockBlobPage.getNextPage()).thenReturn(null);
    when(storage.list(anyString(), any(Storage.BlobListOption[].class))).thenReturn(mockBlobPage);

    runner.getStateManager().setFailOnStateGet(Scope.CLUSTER, true);
    runner.enqueue("test");
    runner.run();

    runner.assertTransferCount(ListGCSBucket.REL_SUCCESS, 0);
    assertEquals(1, runner.getLogger().getErrorMessages().size());
}
 

/** Example of how to disable default event-based hold for a bucket */
public Bucket disableDefaultEventBasedHold(String bucketName) throws StorageException {
  // [START storage_disable_default_event_based_hold]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  Bucket bucket =
      storage.update(BucketInfo.newBuilder(bucketName).setDefaultEventBasedHold(false).build());

  System.out.println("Default event-based hold was disabled for " + bucketName);
  // [END storage_disable_default_event_based_hold]
  return bucket;
}
 

/** Example of how to enable uniform bucket-level access for a bucket */
public Bucket enableUniformBucketLevelAccess(String bucketName) throws StorageException {
  // [START storage_enable_uniform_bucket_level_access]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  BucketInfo.IamConfiguration iamConfiguration =
      BucketInfo.IamConfiguration.newBuilder().setIsUniformBucketLevelAccessEnabled(true).build();
  Bucket bucket =
      storage.update(
          BucketInfo.newBuilder(bucketName).setIamConfiguration(iamConfiguration).build());

  System.out.println("Uniform bucket-level access was enabled for " + bucketName);
  // [END storage_enable_uniform_bucket_level_access]
  return bucket;
}
 

/** Example of deleting an existing inactive HMAC key. * */
public void deleteHmacKey(String accessId, String projectId) throws StorageException {
  // [START storage_delete_hmac_key]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The access ID of the HMAC key, e.g. "GOOG0234230X00"
  // String accessId = "GOOG0234230X00";
  //
  // The ID of the project to which the service account belongs.
  // String projectId = "project-id";
  HmacKeyMetadata metadata =
      storage.getHmacKey(accessId, Storage.GetHmacKeyOption.projectId(projectId));
  storage.deleteHmacKey(metadata);

  System.out.println(
      "The key is deleted, though it will still appear in getHmacKeys() results given showDeletedKey is true.");
  // [END storage_delete_hmac_key]
}
 

/**
 * Copy a blob for gcs to a destination bucket and a path (and delete the source blob if needed)
 * @param sourceBlobId sourceBlobId
 * @param destinationBucket destination bucket
 * @param destinationPath destination path
 * @param deleteSource delete source blob
 */
private void copy(BlobId sourceBlobId, String destinationBucket, String destinationPath, boolean deleteSource) {
  LOG.debug(
      "Copying object '{}' to Object '{}'",
      String.format(BLOB_PATH_TEMPLATE, sourceBlobId.getBucket(), sourceBlobId.getName()),
      String.format(BLOB_PATH_TEMPLATE, destinationBucket, destinationPath)
  );

  Storage.CopyRequest copyRequest = new Storage.CopyRequest.Builder()
      .setSource(sourceBlobId)
      .setTarget(BlobId.of(destinationBucket, destinationPath))
      .build();
  Blob destinationBlob = storage.copy(copyRequest).getResult();
  LOG.debug(
      "Copied object '{}' to Object '{}'",
      String.format(BLOB_PATH_TEMPLATE, sourceBlobId.getBucket(), sourceBlobId.getName()),
      String.format(BLOB_PATH_TEMPLATE, destinationBlob.getBlobId().getBucket(), destinationBlob.getBlobId().getName())
  );
  if (deleteSource) {
    delete(sourceBlobId);
  }
}
 

@AfterClass
public static void tearDown() {
    try {
        // Empty the bucket before deleting it.
        Iterable<Blob> blobIterable = storage.list(BUCKET, Storage.BlobListOption.versions(true)).iterateAll();

        for (final Blob blob : blobIterable) {
            storage.delete(blob.getBlobId());
        }

        storage.delete(BUCKET);
    } catch (final StorageException e) {
        fail("Unable to delete bucket " + BUCKET + ": " + e.getLocalizedMessage());
    }

    if (storage.get(BUCKET) != null) {
        fail("Incomplete teardown, subsequent tests might fail");
    }
}
 

@Test
public void testAclAttributeGroup() throws Exception {
    reset(storage, blob);
    final PutGCSObject processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    when(storage.create(any(BlobInfo.class), any(InputStream.class), any(Storage.BlobWriteOption.class)))
            .thenReturn(blob);

    final Acl.Group mockGroup = mock(Acl.Group.class);
    when(mockGroup.getEmail()).thenReturn(OWNER_GROUP_EMAIL);
    when(blob.getOwner()).thenReturn(mockGroup);

    runner.enqueue("test");
    runner.run();

    runner.assertAllFlowFilesTransferred(PutGCSObject.REL_SUCCESS);
    runner.assertTransferCount(PutGCSObject.REL_SUCCESS, 1);

    final MockFlowFile mockFlowFile = runner.getFlowFilesForRelationship(PutGCSObject.REL_SUCCESS).get(0);
    mockFlowFile.assertAttributeEquals(OWNER_ATTR, OWNER_GROUP_EMAIL);
    mockFlowFile.assertAttributeEquals(OWNER_TYPE_ATTR, "group");
}
 

private void genericTry(final List<List<SinkRecord>> recordBatches,
                        final Integer maxRecordsPerFile) {
    final Storage storage = LocalStorageHelper.getOptions().getService();
    final BucketAccessor testBucketAccessor = new BucketAccessor(storage, TEST_BUCKET, true);

    final Map<String, String> taskProps = basicTaskProps();
    if (maxRecordsPerFile != null) {
        taskProps.put(GcsSinkConfig.FILE_MAX_RECORDS, Integer.toString(maxRecordsPerFile));
    }
    final GcsSinkTask task = new GcsSinkTask(taskProps, storage);

    for (final List<SinkRecord> recordBatch : recordBatches) {
        task.put(recordBatch);
        task.flush(null);
    }

    checkExpectedFileNames(recordBatches, maxRecordsPerFile, testBucketAccessor);
    checkFileSizes(testBucketAccessor, maxRecordsPerFile);
    final int expectedRecordCount = recordBatches.stream().mapToInt(List::size).sum();
    checkTotalRecordCountAndNoMultipleWrites(expectedRecordCount, testBucketAccessor);
    checkTopicPartitionPartInFileNames(testBucketAccessor);
    checkOffsetOrderInFiles(testBucketAccessor);
}
 

public static void changeDefaultStorageClass(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // See the StorageClass documentation for other valid storage classes:
  // https://googleapis.dev/java/google-cloud-clients/latest/com/google/cloud/storage/StorageClass.html
  StorageClass storageClass = StorageClass.COLDLINE;

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Bucket bucket = storage.get(bucketName);
  bucket = bucket.toBuilder().setStorageClass(storageClass).build().update();

  System.out.println(
      "Default storage class for bucket "
          + bucketName
          + " has been set to "
          + bucket.getStorageClass());
}
 

@AfterClass
public static void tearDown() {
    try {
        // Empty the bucket before deleting it.
        Iterator<Blob> blobIterator = storage.list(BUCKET, Storage.BlobListOption.versions(true)).iterateAll();

        while(blobIterator.hasNext()) {
            Blob blob = blobIterator.next();
            storage.delete(blob.getBlobId());
        }

        storage.delete(BUCKET);
    } catch (final StorageException e) {
        fail("Unable to delete bucket " + BUCKET + ": " + e.getLocalizedMessage());
    }

    if (storage.get(BUCKET) != null) {
        fail("Incomplete teardown, subsequent tests might fail");
    }
}
 

public static void downloadEncryptedObject(
    String projectId,
    String bucketName,
    String objectName,
    Path destFilePath,
    String decryptionKey)
    throws IOException {

  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // The ID of your GCS object
  // String objectName = "your-object-name";

  // The path to which the file should be downloaded
  // Path destFilePath = Paths.get("/local/path/to/file.txt");

  // The Base64 encoded decryption key, which should be the same key originally used to encrypt
  // the object
  // String decryptionKey = "TIbv/fjexq+VmtXzAlc63J4z5kFmWJ6NdAPQulQBT7g=";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();

  Blob blob = storage.get(bucketName, objectName);
  blob.downloadTo(destFilePath, Blob.BlobSourceOption.decryptionKey(decryptionKey));

  System.out.println(
      "Downloaded object "
          + objectName
          + " from bucket name "
          + bucketName
          + " to "
          + destFilePath
          + " using customer-supplied encryption key");
}
 

@Test
public void testListOptionsVersions() throws Exception {
    reset(storage, mockBlobPage);
    final ListGCSBucket processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);

    runner.setProperty(ListGCSBucket.USE_GENERATIONS, String.valueOf(USE_GENERATIONS));
    runner.assertValid();

    final Iterable<Blob> mockList = ImmutableList.of();

    when(mockBlobPage.getValues()).thenReturn(mockList);
    when(mockBlobPage.getNextPage()).thenReturn(null);
    when(storage.list(anyString(), argumentCaptor.capture())).thenReturn(mockBlobPage);

    runner.enqueue("test");
    runner.run();

    Storage.BlobListOption option = argumentCaptor.getValue();
    assertEquals(Storage.BlobListOption.versions(true), option);
}
 

@Test
public void testStorageExceptionOnFetch() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    when(storage.get(any(BlobId.class))).thenThrow(new StorageException(400, "test-exception"));
    when(storage.reader(any(BlobId.class), any(Storage.BlobSourceOption.class))).thenReturn(new MockReadChannel(CONTENT));

    runner.enqueue("");

    runner.run();

    runner.assertAllFlowFilesTransferred(FetchGCSObject.REL_FAILURE);
    runner.assertTransferCount(FetchGCSObject.REL_FAILURE, 1);
}
 

@After
public void tearDown() throws IOException {
  ProductManagement.deleteProduct(PROJECT_ID, COMPUTE_REGION, PRODUCT_ID_1);
  ProductSetManagement.deleteProductSet(PROJECT_ID, COMPUTE_REGION, PRODUCT_SET_ID);
  Storage storage = StorageOptions.newBuilder().setProjectId(PROJECT_ID).build().getService();
  // Delete the created blob
  storage.delete(blob.getBlobId());
  System.setOut(null);
}
 

public static void removeBucketIamMember(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // For more information please read:
  // https://cloud.google.com/storage/docs/access-control/iam
  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();

  Policy originalPolicy =
      storage.getIamPolicy(bucketName, Storage.BucketSourceOption.requestedPolicyVersion(3));

  String role = "roles/storage.objectViewer";
  String member = "group:[email protected]";

  // getBindingsList() returns an ImmutableList and copying over to an ArrayList so it's mutable.
  List<Binding> bindings = new ArrayList(originalPolicy.getBindingsList());

  // Remove role-member binding without a condition.
  for (int index = 0; index < bindings.size(); index++) {
    Binding binding = bindings.get(index);
    boolean foundRole = binding.getRole().equals(role);
    boolean foundMember = binding.getMembers().contains(member);
    boolean bindingIsNotConditional = binding.getCondition() == null;

    if (foundRole && foundMember && bindingIsNotConditional) {
      bindings.set(index, binding.toBuilder().removeMembers(member).build());
      break;
    }
  }

  // Update policy to remove member
  Policy.Builder updatedPolicyBuilder = originalPolicy.toBuilder();
  updatedPolicyBuilder.setBindings(bindings).setVersion(3);
  Policy updatedPolicy = storage.setIamPolicy(bucketName, updatedPolicyBuilder.build());

  System.out.printf("Removed %s with role %s from %s\n", member, role, bucketName);
}
 

private Write(Storage storage, long maxBytes, int maxMessages, Duration maxDelay,
    PubsubMessageToTemplatedString batchKeyTemplate, Executor executor,
    Function<Blob, CompletableFuture<Void>> batchCloseHook) {
  super(maxBytes, maxMessages, maxDelay, batchKeyTemplate, executor);
  this.storage = storage;
  this.batchCloseHook = batchCloseHook;
}
 

@Override
public void deleteByTenant(final String tenant) {
    final String folder = sanitizeTenant(tenant);

    LOG.info("Deleting GCS object folder (tenant) from bucket {} and key {}", gcsProperties.getBucketName(),
            folder);
    final Page<Blob> blobs = gcsStorage.list(gcsProperties.getBucketName(),
            Storage.BlobListOption.currentDirectory(), Storage.BlobListOption.prefix(tenant));
    for (final Blob blob : blobs.iterateAll()) {
        gcsStorage.delete(blob.getBlobId());
    }
}
 

private static void cleanUpHmacKeys(ServiceAccount serviceAccount) {
  Page<HmacKeyMetadata> page =
      storage.listHmacKeys(Storage.ListHmacKeysOption.serviceAccount(serviceAccount));
  for (HmacKeyMetadata metadata : page.iterateAll()) {
    if (metadata.getState() == HmacKeyState.ACTIVE) {
      storage.updateHmacKeyState(metadata, HmacKeyState.INACTIVE);
    }
    storage.deleteHmacKey(metadata);
  }
}
 

static void authAppEngineStandard() throws IOException {
  // Explicitly request service account credentials from the app engine standard instance.
  GoogleCredentials credentials = AppEngineCredentials.getApplicationDefault();
  Storage storage = StorageOptions.newBuilder().setCredentials(credentials).build().getService();

  System.out.println("Buckets:");
  Page<Bucket> buckets = storage.list();
  for (Bucket bucket : buckets.iterateAll()) {
    System.out.println(bucket.toString());
  }
}
 

@Test
public void testUploadFileToGcs() {
  Storage storage = mock(Storage.class);
  uploadFileToGcs(
      storage, "my-bucket", Paths.get("my", "filename.txt"), toByteArraySupplier("my data"));
  verify(storage)
      .create(
          BlobInfo.newBuilder("my-bucket", "my/filename.txt")
              .setContentType("text/plain")
              .build(),
          "my data".getBytes(UTF_8));
  verifyNoMoreInteractions(storage);
}
 

GcsArtifact(final Storage gcsStorage, final GcsRepositoryProperties gcsProperties, final String key,
        final String artifactId, final DbArtifactHash hashes, final Long size, final String contentType) {
    super(artifactId, hashes, size, contentType);
    Assert.notNull(gcsStorage, "GCS cannot be null");
    Assert.notNull(gcsProperties, "Properties cannot be null");
    Assert.notNull(key, "Key cannot be null");
    this.gcsStorage = gcsStorage;
    this.gcsProperties = gcsProperties;
    this.key = key;
}
 

@Override
public PutGCSObject getProcessor() {
    return new PutGCSObject() {
        @Override
        protected Storage getCloudService() {
            return storage;
        }
    };
}
 

private void uploadResultsToGcs(String destination) {
  checkArgument(
      !destination.isEmpty(), "destination must include at least the bucket name, but is empty");
  Path bucketWithFolder = Paths.get(destination, createUniqueFolderName());
  String bucket = bucketWithFolder.getName(0).toString();
  Path folder = bucketWithFolder.subpath(1, bucketWithFolder.getNameCount());

  StorageOptions.Builder storageOptions = StorageOptions.newBuilder();
  if (!isNullOrEmpty(credentialsFile)) {
    try {
      storageOptions.setCredentials(
          GoogleCredentials.fromStream(new FileInputStream(credentialsFile)));
    } catch (IOException e) {
      throw new UncheckedIOException(e);
    }
  }
  Storage storage = storageOptions.build().getService();

  FilesWithEntryPoint filesToUpload = generateFilesToUpload();

  System.out.format(
      "ReportUploader: going to upload %s files to %s/%s\n",
      filesToUpload.files().size(), bucket, folder);
  if ("yes".equals(multithreadedUpload)) {
    System.out.format("ReportUploader: multi-threaded upload\n");
    uploadFilesToGcsMultithread(storage, bucket, folder, filesToUpload.files());
  } else {
    System.out.format("ReportUploader: single threaded upload\n");
    filesToUpload
        .files()
        .forEach(
            (path, dataSupplier) -> {
              System.out.format("ReportUploader: Uploading %s\n", path);
              uploadFileToGcs(storage, bucket, folder.resolve(path), dataSupplier);
            });
  }
  System.out.format(
      "ReportUploader: report uploaded to https://storage.googleapis.com/%s/%s\n",
      bucket, folder.resolve(filesToUpload.entryPoint()));
}