com.google.cloud.storage.BucketInfo Java Examples

/** Example of setting a default KMS key on a bucket. */
public Bucket setDefaultKmsKey(String bucketName, String kmsKeyName) throws StorageException {
  // [START storage_set_bucket_default_kms_key]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of the existing bucket to set a default KMS key for, e.g. "my-bucket"
  // String bucketName = "my-bucket"

  // The name of the KMS-key to use as a default
  // Key names are provided in the following format:
  // 'projects/<PROJECT>/locations/<LOCATION>/keyRings/<RING_NAME>/cryptoKeys/<KEY_NAME>'
  // String kmsKeyName = ""

  BucketInfo bucketInfo =
      BucketInfo.newBuilder(bucketName).setDefaultKmsKeyName(kmsKeyName).build();

  Bucket bucket = storage.update(bucketInfo);

  System.out.println("Default KMS Key Name: " + bucket.getDefaultKmsKeyName());
  // [END storage_set_bucket_default_kms_key]
  return bucket;
}
 

@BeforeClass
public static void setUp() {
    try {
        helper = RemoteStorageHelper.create();
        storage = helper.getOptions().getService();

        if (storage.get(BUCKET) != null) {
            // As the generateBucketName function uses a UUID, this should pretty much never happen
            fail("Bucket " + BUCKET + " exists. Please rerun the test to generate a new bucket name.");
        }

        // Create the bucket
        storage.create(BucketInfo.of(BUCKET));
    } catch (StorageException e) {
        fail("Can't create bucket " + BUCKET + ": " + e.getLocalizedMessage());
    }

    if (storage.get(BUCKET) == null) {
        fail("Setup incomplete, tests will fail");
    }
}
 

@BeforeClass
public static void setUp() {
    try {
        helper = RemoteStorageHelper.create();
        storage = helper.getOptions().getService();

        if (storage.get(BUCKET) != null) {
            // As the generateBucketName function uses a UUID, this should pretty much never happen
            fail("Bucket " + BUCKET + " exists. Please rerun the test to generate a new bucket name.");
        }

        // Create the bucket
        storage.create(BucketInfo.of(BUCKET));
    } catch (StorageException e) {
        fail("Can't create bucket " + BUCKET + ": " + e.getLocalizedMessage());
    }

    if (storage.get(BUCKET) == null) {
        fail("Setup incomplete, tests will fail");
    }
}
 

@Test
public void testMoveObject() {
  String blob = "movethisblob";
  storage.create(BlobInfo.newBuilder(BlobId.of(BUCKET, blob)).build());
  assertNotNull(storage.get(BUCKET, blob));
  String newBucket = RemoteStorageHelper.generateBucketName();
  storage.create(BucketInfo.newBuilder(newBucket).build());
  try {
    MoveObject.moveObject(PROJECT_ID, BUCKET, blob, newBucket);
    assertNotNull(storage.get(newBucket, blob));
    assertNull(storage.get(BUCKET, blob));
  } finally {
    storage.delete(newBucket, blob);
    storage.delete(newBucket);
  }
}
 

/** Example of setting a retention policy on a bucket */
public Bucket setRetentionPolicy(String bucketName, Long retentionPeriod)
    throws StorageException {
  // [START storage_set_retention_policy]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  // The retention period for objects in bucket
  // Long retentionPeriod = 3600L; // 1 hour in seconds

  Bucket bucketWithRetentionPolicy =
      storage.update(
          BucketInfo.newBuilder(bucketName).setRetentionPeriod(retentionPeriod).build());

  System.out.println(
      "Retention period for "
          + bucketName
          + " is now "
          + bucketWithRetentionPolicy.getRetentionPeriod());
  // [END storage_set_retention_policy]
  return bucketWithRetentionPolicy;
}
 

/** Example of how to enable default event-based hold for a bucket */
public Bucket enableDefaultEventBasedHold(String bucketName) throws StorageException {
  // [START storage_enable_default_event_based_hold]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  Bucket bucket =
      storage.update(BucketInfo.newBuilder(bucketName).setDefaultEventBasedHold(true).build());

  System.out.println("Default event-based hold was enabled for " + bucketName);
  // [END storage_enable_default_event_based_hold]
  return bucket;
}
 

/** Example of how to disable default event-based hold for a bucket */
public Bucket disableDefaultEventBasedHold(String bucketName) throws StorageException {
  // [START storage_disable_default_event_based_hold]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  Bucket bucket =
      storage.update(BucketInfo.newBuilder(bucketName).setDefaultEventBasedHold(false).build());

  System.out.println("Default event-based hold was disabled for " + bucketName);
  // [END storage_disable_default_event_based_hold]
  return bucket;
}
 

/** Example of how to enable uniform bucket-level access for a bucket */
public Bucket enableUniformBucketLevelAccess(String bucketName) throws StorageException {
  // [START storage_enable_uniform_bucket_level_access]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  BucketInfo.IamConfiguration iamConfiguration =
      BucketInfo.IamConfiguration.newBuilder().setIsUniformBucketLevelAccessEnabled(true).build();
  Bucket bucket =
      storage.update(
          BucketInfo.newBuilder(bucketName).setIamConfiguration(iamConfiguration).build());

  System.out.println("Uniform bucket-level access was enabled for " + bucketName);
  // [END storage_enable_uniform_bucket_level_access]
  return bucket;
}
 

/** Example of how to disable uniform bucket-level access for a bucket */
public Bucket disableUniformBucketLevelAccess(String bucketName) throws StorageException {
  // [START storage_disable_uniform_bucket_level_access]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  BucketInfo.IamConfiguration iamConfiguration =
      BucketInfo.IamConfiguration.newBuilder()
          .setIsUniformBucketLevelAccessEnabled(false)
          .build();
  Bucket bucket =
      storage.update(
          BucketInfo.newBuilder(bucketName).setIamConfiguration(iamConfiguration).build());

  System.out.println("Uniform bucket-level access was disabled for " + bucketName);
  // [END storage_disable_uniform_bucket_level_access]
  return bucket;
}
 

/** Example of how to get uniform bucket-level access metadata for a bucket */
public Bucket getUniformBucketLevelAccess(String bucketName) throws StorageException {
  // [START storage_get_uniform_bucket_level_access]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  Bucket bucket = storage.get(bucketName, BucketGetOption.fields(BucketField.IAMCONFIGURATION));
  BucketInfo.IamConfiguration iamConfiguration = bucket.getIamConfiguration();

  Boolean enabled = iamConfiguration.isUniformBucketLevelAccessEnabled();
  Date lockedTime = new Date(iamConfiguration.getUniformBucketLevelAccessLockedTime());

  if (enabled != null && enabled) {
    System.out.println("Uniform bucket-level access is enabled for " + bucketName);
    System.out.println("Bucket will be locked on " + lockedTime);
  } else {
    System.out.println("Uniform bucket-level access is disabled for " + bucketName);
  }
  // [END storage_get_uniform_bucket_level_access]
  return bucket;
}
 

@BeforeClass
public static void beforeClass() {
  bigquery = RemoteBigQueryHelper.create().getOptions().getService();
  bigquery.create(DatasetInfo.newBuilder(DATASET_NAME).build());
  bigquery.create(DatasetInfo.newBuilder(COPY_DATASET_NAME).build());
  storage = RemoteStorageHelper.create().getOptions().getService();
  storage.create(BucketInfo.of(BUCKET_NAME));
}
 

public static void main(String... args) {
  // Create a service object
  // Credentials are inferred from the environment.
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // Create a bucket
  String bucketName = "my_unique_bucket"; // Change this to something unique
  Bucket bucket = storage.create(BucketInfo.of(bucketName));

  // Upload a blob to the newly created bucket
  Blob blob = bucket.create("my_blob_name", "a simple blob".getBytes(UTF_8), "text/plain");

  // Read the blob content from the server
  String blobContent = new String(blob.getContent(), UTF_8);

  // List all your buckets
  System.out.println("My buckets:");
  for (Bucket currentBucket : storage.list().iterateAll()) {
    System.out.println(currentBucket);
  }

  // List the blobs in a particular bucket
  System.out.println("My blobs:");
  for (Blob currentBlob : bucket.list().iterateAll()) {
    System.out.println(currentBlob);
  }
}
 

/** Example of updating bucket information. */
// [TARGET update(BucketInfo, BucketTargetOption...)]
// [VARIABLE "my_unique_bucket"]
public Bucket updateBucket(String bucketName) {
  // [START updateBucket]
  BucketInfo bucketInfo = BucketInfo.newBuilder(bucketName).setVersioningEnabled(true).build();
  Bucket bucket = storage.update(bucketInfo);
  // [END updateBucket]
  return bucket;
}
 

@Test
public void testDeleteBucket() {
  String newBucket = RemoteStorageHelper.generateBucketName();
  storage.create(BucketInfo.newBuilder(newBucket).build());
  assertNotNull(storage.get(newBucket));
  try {
    DeleteBucket.deleteBucket(PROJECT_ID, newBucket);
    assertNull(storage.get(newBucket));
  } finally {
    storage.delete(newBucket);
  }
}
 

@Test
public void testAddListRemoveBucketIamMembers() {
  storage.update(
      BucketInfo.newBuilder(BUCKET)
          .setIamConfiguration(
              BucketInfo.IamConfiguration.newBuilder()
                  .setIsUniformBucketLevelAccessEnabled(true)
                  .build())
          .build());
  int originalSize = storage.getIamPolicy(BUCKET).getBindingsList().size();
  AddBucketIamMember.addBucketIamMember(PROJECT_ID, BUCKET);
  assertEquals(originalSize + 1, storage.getIamPolicy(BUCKET).getBindingsList().size());
  PrintStream standardOut = System.out;
  final ByteArrayOutputStream snippetOutputCapture = new ByteArrayOutputStream();
  System.setOut(new PrintStream(snippetOutputCapture));
  ListBucketIamMembers.listBucketIamMembers(PROJECT_ID, BUCKET);
  String snippetOutput = snippetOutputCapture.toString();
  System.setOut(standardOut);
  assertTrue(snippetOutput.contains("[email protected]"));
  RemoveBucketIamMember.removeBucketIamMember(PROJECT_ID, BUCKET);
  assertEquals(originalSize, storage.getIamPolicy(BUCKET).getBindingsList().size());
  AddBucketIamConditionalBinding.addBucketIamConditionalBinding(PROJECT_ID, BUCKET);
  assertEquals(originalSize + 1, storage.getIamPolicy(BUCKET).getBindingsList().size());
  RemoveBucketIamConditionalBinding.removeBucketIamConditionalBinding(PROJECT_ID, BUCKET);
  assertEquals(originalSize, storage.getIamPolicy(BUCKET).getBindingsList().size());
  storage.update(
      BucketInfo.newBuilder(BUCKET)
          .setIamConfiguration(
              BucketInfo.IamConfiguration.newBuilder()
                  .setIsUniformBucketLevelAccessEnabled(false)
                  .build())
          .build());
}
 

@Test
public void testCopyObject() {
  String newBucket = RemoteStorageHelper.generateBucketName();
  storage.create(BucketInfo.newBuilder(newBucket).build());
  try {
    CopyObject.copyObject(PROJECT_ID, BUCKET, BLOB, newBucket);
    assertNotNull(storage.get(newBucket, BLOB));
  } finally {
    storage.delete(newBucket, BLOB);
    storage.delete(newBucket);
  }
}
 

@Test
public void testSimpleListWithPrefixAndGenerations() throws Exception {
    // enable versioning
    storage.update(BucketInfo.newBuilder(BUCKET).setVersioningEnabled(true).build());

    putTestFile("generations/a", CONTENT);
    putTestFile("generations/a", CONTENT);
    putTestFile("generations/b", CONTENT);
    putTestFile("generations/c", CONTENT);

    final TestRunner runner = buildNewRunner(new ListGCSBucket());
    runner.setProperty(ListGCSBucket.BUCKET, BUCKET);

    runner.setProperty(ListGCSBucket.PREFIX, "generations/");
    runner.setProperty(ListGCSBucket.USE_GENERATIONS, "true");

    runner.run();

    runner.assertAllFlowFilesTransferred(ListGCSBucket.REL_SUCCESS, 4);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListGCSBucket.REL_SUCCESS);
    flowFiles.get(0).assertAttributeEquals("filename", "generations/a");
    flowFiles.get(1).assertAttributeEquals("filename", "generations/a");
    flowFiles.get(2).assertAttributeEquals("filename", "generations/b");
    flowFiles.get(3).assertAttributeEquals("filename", "generations/c");

    assertNotEquals(
            flowFiles.get(0).getAttribute(StorageAttributes.GENERATION_ATTR),
            flowFiles.get(1).getAttribute(StorageAttributes.GENERATION_ATTR)
    );
}
 

/** Example of creating a bucket. */
// [TARGET create(BucketInfo, BucketTargetOption...)]
// [VARIABLE "my_unique_bucket"]
public Bucket createBucket(String bucketName) {
  // [START createBucket]
  Bucket bucket = storage.create(BucketInfo.of(bucketName));
  // [END createBucket]
  return bucket;
}
 

@Override
public void create(final String bucketName) {
    try {
        if (!doesExist(bucketName)) {
            storage.create(BucketInfo.of(bucketName));
        }
    } catch (final StorageException ex) {
        if (ex.getCode() == 409 && ex.getMessage().contains("You already own this bucket.")) {
            logger.warn(format("Unable to create bucket %s: %s", bucketName, ex.getMessage()));
        } else {
            throw new GCPModuleException(format("Unable to create bucket %s", bucketName), ex);
        }
    }
}
 

@Test
public void testSimpleListWithPrefixAndGenerations() throws Exception {
    // enable versioning
    storage.update(BucketInfo.newBuilder(BUCKET).setVersioningEnabled(true).build());

    putTestFile("generations/a", CONTENT);
    putTestFile("generations/a", CONTENT);
    putTestFile("generations/b", CONTENT);
    putTestFile("generations/c", CONTENT);

    final TestRunner runner = buildNewRunner(new ListGCSBucket());
    runner.setProperty(ListGCSBucket.BUCKET, BUCKET);

    runner.setProperty(ListGCSBucket.PREFIX, "generations/");
    runner.setProperty(ListGCSBucket.USE_GENERATIONS, "true");

    runner.run();

    runner.assertAllFlowFilesTransferred(ListGCSBucket.REL_SUCCESS, 4);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListGCSBucket.REL_SUCCESS);
    flowFiles.get(0).assertAttributeEquals("filename", "generations/a");
    flowFiles.get(1).assertAttributeEquals("filename", "generations/a");
    flowFiles.get(2).assertAttributeEquals("filename", "generations/b");
    flowFiles.get(3).assertAttributeEquals("filename", "generations/c");

    assertNotEquals(
            flowFiles.get(0).getAttribute(StorageAttributes.GENERATION_ATTR),
            flowFiles.get(1).getAttribute(StorageAttributes.GENERATION_ATTR)
    );
}
 

/** Find credentials in the environment and create a bucket in GCS. */
@Override
protected void starting(Description description) {
  RemoteStorageHelper storageHelper = RemoteStorageHelper.create();
  storage = storageHelper.getOptions().getService();
  bucket = RemoteStorageHelper.generateBucketName();
  storage.create(BucketInfo.newBuilder(bucket).build());
}
 

/** Create a storage bucket for metadata and keys. */
@Before
public void createBucket() {
  RemoteStorageHelper storageHelper = RemoteStorageHelper.create();
  storage = storageHelper.getOptions().getService();
  projectId = storageHelper.getOptions().getProjectId();
  bucket = RemoteStorageHelper.generateBucketName();
  storage.create(BucketInfo.of(bucket));
}
 

/** Find credentials in the environment and create a bucket in GCS. */
@Before
public void createBucket() {
  RemoteStorageHelper storageHelper = RemoteStorageHelper.create();
  storage = storageHelper.getOptions().getService();
  projectId = storageHelper.getOptions().getProjectId();
  bucket = RemoteStorageHelper.generateBucketName();
  storage.create(BucketInfo.of(bucket));
}
 

public Bucket createBucket(String bucketName) {
  Bucket bucket = storage.create(
      BucketInfo.newBuilder(bucketName)
          .setLocation("europe-west1")
          .build());
  return bucket;
}
 

@Override
public boolean mkdir(String directory) throws IOException {
	try {
		this.gcs.create(BucketInfo.of(directory));
		return true;
	}
	catch (StorageException se) {
		LOGGER.info("Error creating the GCS bucket.", se);
		return false;
	}

}
 

@Test
public void testBucketDoesNotExist() {
	Bucket mockedBucket = mock(Bucket.class);
	when(this.mockStorage.create(BucketInfo.newBuilder("non-existing").build())).thenReturn(mockedBucket);
	when(mockedBucket.getName()).thenReturn("test-spring");

	GoogleStorageResource bucket = new GoogleStorageResource(this.mockStorage, "gs://non-existing/");
	Assert.assertFalse(bucket.bucketExists());
	Assert.assertFalse(bucket.exists());

	Assert.assertNotNull(bucket.createBucket());
}
 

@Before
public void setUp() {
  bout = new ByteArrayOutputStream();
  System.setOut(new PrintStream(bout));

  Storage storage = StorageOptions.getDefaultInstance().getService();
  bucket = storage.create(BucketInfo.of(BUCKET_NAME));
  blob = bucket.create(JOB_FILE_NAME, SORT_CODE.getBytes(UTF_8), "text/plain");
}
 

public static void main(String... args) throws Exception {
  // Instantiates a client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name for the new bucket
  String bucketName = args[0];  // "my-new-bucket";

  // Creates the new bucket
  Bucket bucket = storage.create(BucketInfo.of(bucketName));

  System.out.printf("Bucket %s created.%n", bucket.getName());
}
 

public static void createBucketWithStorageClassAndLocation(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID to give your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();

  // See the StorageClass documentation for other valid storage classes:
  // https://googleapis.dev/java/google-cloud-clients/latest/com/google/cloud/storage/StorageClass.html
  StorageClass storageClass = StorageClass.COLDLINE;

  // See this documentation for other valid locations:
  // http://g.co/cloud/storage/docs/bucket-locations#location-mr
  String location = "asia";

  Bucket bucket =
      storage.create(
          BucketInfo.newBuilder(bucketName)
              .setStorageClass(storageClass)
              .setLocation(location)
              .build());

  System.out.println(
      "Created bucket "
          + bucket.getName()
          + " in "
          + bucket.getLocation()
          + " with storage class "
          + bucket.getStorageClass());
}
 

protected Bucket getOrCreateStorageBucket(final String location) {
  Bucket bucket = storage.get(getConfiguredBucketName());
  if (bucket == null) {
    bucket = storage.create(
        BucketInfo.newBuilder(getConfiguredBucketName())
            .setLocation(location)
            .setStorageClass(StorageClass.REGIONAL)
            .build());
  }

  return bucket;
}