Java Code Examples for com.google.cloud.storage.Storage#get()

/** Example of removing a retention policy on a bucket */
public Bucket removeRetentionPolicy(String bucketName)
    throws StorageException, IllegalArgumentException {
  // [START storage_remove_retention_policy]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  Bucket bucket = storage.get(bucketName, BucketGetOption.fields(BucketField.RETENTION_POLICY));
  if (bucket.retentionPolicyIsLocked() != null && bucket.retentionPolicyIsLocked()) {
    throw new IllegalArgumentException(
        "Unable to remove retention period as retention policy is locked.");
  }

  Bucket bucketWithoutRetentionPolicy =
      bucket.toBuilder().setRetentionPeriod(null).build().update();

  System.out.println("Retention period for " + bucketName + " has been removed");
  // [END storage_remove_retention_policy]
  return bucketWithoutRetentionPolicy;
}
 

public static void setObjectMetadata(String projectId, String bucketName, String objectName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // The ID of your GCS object
  // String objectName = "your-object-name";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Map<String, String> newMetadata = new HashMap<>();
  newMetadata.put("keyToAddOrUpdate", "value");
  Blob blob = storage.get(bucketName, objectName);
  // Does an upsert operation, if the key already exists it's replaced by the new value, otherwise
  // it's added.
  blob.toBuilder().setMetadata(newMetadata).build().update();

  System.out.println(
      "Updated custom metadata for object " + objectName + " in bucket " + bucketName);
}
 

public static void downloadRequesterPaysObject(
    String projectId, String bucketName, String objectName, Path destFilePath) {
  // The project ID to bill
  // String projectId = "my-billable-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // The ID of your GCS object
  // String objectName = "your-object-name";

  // The path to which the file should be downloaded
  // Path destFilePath = Paths.get("/local/path/to/file.txt");

  Storage storage = StorageOptions.getDefaultInstance().getService();
  Blob blob = storage.get(BlobId.of(bucketName, objectName));
  blob.downloadTo(destFilePath, Blob.BlobSourceOption.userProject(projectId));

  System.out.println(
      "Object " + objectName + " downloaded to " + destFilePath + " and billed to " + projectId);
}
 

public static void enableLifecycleManagement(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Bucket bucket = storage.get(bucketName);

  // See the LifecycleRule documentation for additional info on what you can do with lifecycle
  // management rules. This one deletes objects that are over 100 days old.
  // https://googleapis.dev/java/google-cloud-clients/latest/com/google/cloud/storage/BucketInfo.LifecycleRule.html
  bucket
      .toBuilder()
      .setLifecycleRules(
          ImmutableList.of(
              new LifecycleRule(
                  LifecycleRule.LifecycleAction.newDeleteAction(),
                  LifecycleRule.LifecycleCondition.newBuilder().setAge(100).build())))
      .build()
      .update();

  System.out.println("Lifecycle management was enabled and configured for bucket " + bucketName);
}
 

/** Example of how to lock a bucket retention policy */
public Bucket lockRetentionPolicy(String bucketName) throws StorageException {
  // [START storage_lock_retention_policy]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  Bucket bucket =
      storage.get(bucketName, Storage.BucketGetOption.fields(BucketField.METAGENERATION));
  Bucket lockedBucket =
      bucket.lockRetentionPolicy(Storage.BucketTargetOption.metagenerationMatch());

  System.out.println("Retention period for " + bucketName + " is now locked");
  System.out.println(
      "Retention policy effective as of " + new Date(lockedBucket.getRetentionEffectiveTime()));
  // [END storage_lock_retention_policy]
  return lockedBucket;
}
 

/** Example of how to get a bucket's retention policy */
public Bucket getRetentionPolicy(String bucketName) throws StorageException {
  // [START storage_get_retention_policy]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  Bucket bucket = storage.get(bucketName, BucketGetOption.fields(BucketField.RETENTION_POLICY));

  System.out.println("Retention Policy for " + bucketName);
  System.out.println("Retention Period: " + bucket.getRetentionPeriod());
  if (bucket.retentionPolicyIsLocked() != null && bucket.retentionPolicyIsLocked()) {
    System.out.println("Retention Policy is locked");
  }
  if (bucket.getRetentionEffectiveTime() != null) {
    System.out.println("Effective Time: " + new Date(bucket.getRetentionEffectiveTime()));
  }
  // [END storage_get_retention_policy]
  return bucket;
}
 

public static void changeDefaultStorageClass(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // See the StorageClass documentation for other valid storage classes:
  // https://googleapis.dev/java/google-cloud-clients/latest/com/google/cloud/storage/StorageClass.html
  StorageClass storageClass = StorageClass.COLDLINE;

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Bucket bucket = storage.get(bucketName);
  bucket = bucket.toBuilder().setStorageClass(storageClass).build().update();

  System.out.println(
      "Default storage class for bucket "
          + bucketName
          + " has been set to "
          + bucket.getStorageClass());
}
 

public static void downloadEncryptedObject(
    String projectId,
    String bucketName,
    String objectName,
    Path destFilePath,
    String decryptionKey)
    throws IOException {

  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // The ID of your GCS object
  // String objectName = "your-object-name";

  // The path to which the file should be downloaded
  // Path destFilePath = Paths.get("/local/path/to/file.txt");

  // The Base64 encoded decryption key, which should be the same key originally used to encrypt
  // the object
  // String decryptionKey = "TIbv/fjexq+VmtXzAlc63J4z5kFmWJ6NdAPQulQBT7g=";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();

  Blob blob = storage.get(bucketName, objectName);
  blob.downloadTo(destFilePath, Blob.BlobSourceOption.decryptionKey(decryptionKey));

  System.out.println(
      "Downloaded object "
          + objectName
          + " from bucket name "
          + bucketName
          + " to "
          + destFilePath
          + " using customer-supplied encryption key");
}
 

public static void downloadPublicObject(
    String bucketName, String publicObjectName, Path destFilePath) {
  // The name of the bucket to access
  // String bucketName = "my-bucket";

  // The name of the remote public file to download
  // String publicObjectName = "publicfile.txt";

  // The path to which the file should be downloaded
  // Path destFilePath = Paths.get("/local/path/to/file.txt");

  // Instantiate an anonymous Google Cloud Storage client, which can only access public files
  Storage storage = StorageOptions.getUnauthenticatedInstance().getService();

  Blob blob = storage.get(BlobId.of(bucketName, publicObjectName));
  blob.downloadTo(destFilePath);

  System.out.println(
      "Downloaded public object "
          + publicObjectName
          + " from bucket name "
          + bucketName
          + " to "
          + destFilePath);
}
 

public static void listObjectsWithOldVersions(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";
  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Bucket bucket = storage.get(bucketName);
  Page<Blob> blobs = bucket.list(Storage.BlobListOption.versions(true));

  for (Blob blob : blobs.iterateAll()) {
    System.out.println(blob.getName() + "," + blob.getGeneration());
  }
}
 

public static void enableRequesterPays(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Bucket bucket = storage.get(bucketName);
  bucket.toBuilder().setRequesterPays(true).build().update();

  System.out.println("Requester pays enabled for bucket " + bucketName);
}
 

@Provides
@Singleton
Bucket getBucket(@ProjectId String projectId) {
  Storage storage = StorageOptions.getDefaultInstance().getService();
  // Must match BUCKET_NAME for user data bucket in setup_gke_environment.sh
  String bucketId = "user-data-" + projectId;
  return storage.get(bucketId);
}
 

public static void disableBucketVersioning(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Bucket bucket = storage.get(bucketName);
  bucket.toBuilder().setVersioningEnabled(false).build().update();

  System.out.println("Versioning is now disabled for bucket " + bucketName);
}
 

public static void enableBucketVersioning(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Bucket bucket = storage.get(bucketName);
  bucket.toBuilder().setVersioningEnabled(true).build().update();

  System.out.println("Versioning is now enabled for bucket " + bucketName);
}
 

public static void removeBucketDefaultKmsKey(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Bucket bucket = storage.get(bucketName);
  bucket.toBuilder().setDefaultKmsKeyName(null).build().update();

  System.out.println("Default KMS key was removed from " + bucketName);
}
 

public static void moveObject(
    String projectId, String sourceBucketName, String objectName, String targetBucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // The ID of your GCS object
  // String objectName = "your-object-name";

  // The ID of the bucket to move the object objectName to
  // String targetBucketName = "target-object-bucket"

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Blob blob = storage.get(sourceBucketName, objectName);
  // Write a copy of the object to the target bucket
  CopyWriter copyWriter = blob.copyTo(targetBucketName, objectName);
  Blob copiedBlob = copyWriter.getResult();
  // Delete the original blob now that we've copied to where we want it, finishing the "move"
  // operation
  blob.delete();

  System.out.println(
      "Moved object "
          + objectName
          + " from bucket "
          + sourceBucketName
          + " to "
          + copiedBlob.getBucket());
}
 

public static void configureBucketCors(
    String projectId,
    String bucketName,
    String origin,
    String responseHeader,
    Integer maxAgeSeconds) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // The origin for this CORS config to allow requests from
  // String origin = "http://example.appspot.com";

  // The response header to share across origins
  // String responseHeader = "Content-Type";

  // The maximum amount of time the browser can make requests before it must repeat preflighted
  // requests
  // Integer maxAgeSeconds = 3600;

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Bucket bucket = storage.get(bucketName);

  // See the HttpMethod documentation for other HTTP methods available:
  // https://cloud.google.com/appengine/docs/standard/java/javadoc/com/google/appengine/api/urlfetch/HTTPMethod
  HttpMethod method = HttpMethod.GET;

  Cors cors =
      Cors.newBuilder()
          .setOrigins(ImmutableList.of(Cors.Origin.of(origin)))
          .setMethods(ImmutableList.of(method))
          .setResponseHeaders(ImmutableList.of(responseHeader))
          .setMaxAgeSeconds(maxAgeSeconds)
          .build();

  bucket.toBuilder().setCors(ImmutableList.of(cors)).build().update();

  System.out.println(
      "Bucket "
          + bucketName
          + " was updated with a CORS config to allow GET requests from "
          + origin
          + " sharing "
          + responseHeader
          + " responses across origins");
}
 

public static void listObjectsWithPrefix(
    String projectId, String bucketName, String directoryPrefix) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // The directory prefix to search for
  // String directoryPrefix = "myDirectory/"

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Bucket bucket = storage.get(bucketName);
  /**
   * Using the Storage.BlobListOption.currentDirectory() option here causes the results to display
   * in a "directory-like" mode, showing what objects are in the directory you've specified, as
   * well as what other directories exist in that directory. For example, given these blobs:
   *
   * <p>a/1.txt a/b/2.txt a/b/3.txt
   *
   * <p>If you specify prefix = "a/" and don't use Storage.BlobListOption.currentDirectory(),
   * you'll get back:
   *
   * <p>a/1.txt a/b/2.txt a/b/3.txt
   *
   * <p>However, if you specify prefix = "a/" and do use
   * Storage.BlobListOption.currentDirectory(), you'll get back:
   *
   * <p>a/1.txt a/b/
   *
   * <p>Because a/1.txt is the only file in the a/ directory and a/b/ is a directory inside the
   * /a/ directory.
   */
  Page<Blob> blobs =
      bucket.list(
          Storage.BlobListOption.prefix(directoryPrefix),
          Storage.BlobListOption.currentDirectory());

  for (Blob blob : blobs.iterateAll()) {
    System.out.println(blob.getName());
  }
}
 

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final long startNanos = System.nanoTime();

    final String bucketName = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue();
    final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue();
    final Long generation = context.getProperty(GENERATION).evaluateAttributeExpressions(flowFile).asLong();
    final String encryptionKey = context.getProperty(ENCRYPTION_KEY).evaluateAttributeExpressions(flowFile).getValue();

    final Storage storage = getCloudService();
    final BlobId blobId = BlobId.of(bucketName, key, generation);

    try {
        final List<Storage.BlobSourceOption> blobSourceOptions = new ArrayList<>(2);

        if (encryptionKey != null) {
            blobSourceOptions.add(Storage.BlobSourceOption.decryptionKey(encryptionKey));
        }

        if (generation != null) {
            blobSourceOptions.add(Storage.BlobSourceOption.generationMatch());
        }

        final Blob blob = storage.get(blobId);
        if (blob == null) {
            throw new StorageException(404, "Blob " + blobId + " not found");
        }

        final ReadChannel reader = storage.reader(blobId, blobSourceOptions.toArray(new Storage.BlobSourceOption[0]));
        flowFile = session.importFrom(Channels.newInputStream(reader), flowFile);

        final Map<String, String> attributes = StorageAttributes.createAttributes(blob);
        flowFile = session.putAllAttributes(flowFile, attributes);
    } catch (StorageException e) {
        getLogger().error("Failed to fetch GCS Object due to {}", new Object[] {e}, e);
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }

    session.transfer(flowFile, REL_SUCCESS);

    final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    getLogger().info("Successfully retrieved GCS Object for {} in {} millis; routing to success", new Object[]{flowFile, millis});
    session.getProvenanceReporter().fetch(flowFile, "https://" + bucketName + ".storage.googleapis.com/" + key, millis);
}
 

public static void getObjectMetadata(String projectId, String bucketName, String blobName)
    throws StorageException {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // The ID of your GCS object
  // String objectName = "your-object-name";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();

  // Select all fields
  // Fields can be selected individually e.g. Storage.BlobField.CACHE_CONTROL
  Blob blob =
      storage.get(bucketName, blobName, Storage.BlobGetOption.fields(Storage.BlobField.values()));

  // Print blob metadata
  System.out.println("Bucket: " + blob.getBucket());
  System.out.println("CacheControl: " + blob.getCacheControl());
  System.out.println("ComponentCount: " + blob.getComponentCount());
  System.out.println("ContentDisposition: " + blob.getContentDisposition());
  System.out.println("ContentEncoding: " + blob.getContentEncoding());
  System.out.println("ContentLanguage: " + blob.getContentLanguage());
  System.out.println("ContentType: " + blob.getContentType());
  System.out.println("Crc32c: " + blob.getCrc32c());
  System.out.println("Crc32cHexString: " + blob.getCrc32cToHexString());
  System.out.println("ETag: " + blob.getEtag());
  System.out.println("Generation: " + blob.getGeneration());
  System.out.println("Id: " + blob.getBlobId());
  System.out.println("KmsKeyName: " + blob.getKmsKeyName());
  System.out.println("Md5Hash: " + blob.getMd5());
  System.out.println("Md5HexString: " + blob.getMd5ToHexString());
  System.out.println("MediaLink: " + blob.getMediaLink());
  System.out.println("Metageneration: " + blob.getMetageneration());
  System.out.println("Name: " + blob.getName());
  System.out.println("Size: " + blob.getSize());
  System.out.println("StorageClass: " + blob.getStorageClass());
  System.out.println("TimeCreated: " + new Date(blob.getCreateTime()));
  System.out.println("Last Metadata Update: " + new Date(blob.getUpdateTime()));
  Boolean temporaryHoldIsEnabled = (blob.getTemporaryHold() != null && blob.getTemporaryHold());
  System.out.println("temporaryHold: " + (temporaryHoldIsEnabled ? "enabled" : "disabled"));
  Boolean eventBasedHoldIsEnabled =
      (blob.getEventBasedHold() != null && blob.getEventBasedHold());
  System.out.println("eventBasedHold: " + (eventBasedHoldIsEnabled ? "enabled" : "disabled"));
  if (blob.getRetentionExpirationTime() != null) {
    System.out.println("retentionExpirationTime: " + new Date(blob.getRetentionExpirationTime()));
  }
  if (blob.getMetadata() != null) {
    System.out.println("\n\n\nUser metadata:");
    for (Map.Entry<String, String> userMetadata : blob.getMetadata().entrySet()) {
      System.out.println(userMetadata.getKey() + "=" + userMetadata.getValue());
    }
  }
}