com.google.cloud.storage.BlobId Java Examples

private void storeMessagesInGCS(final List<Record> errorRecords) throws StorageException {
    //get all messages to serialize per topic
    final Map<String, GCSInvalidMessagesWrapper> topicMessagesMap = getMessagesToSerializePerTopic(errorRecords);
    //serialize the messages in GCS for each topic - a file with all messages per topic is stored in GCS
    topicMessagesMap.keySet().forEach(topicName -> {
        final String fileName = UUID.randomUUID().toString();
        final String pathPrefix = gcsBasePathPrefix + "/" + topicName + "/" + getFormattedDatePrefix(Instant.now()) + "/";
        final BlobId blobId = BlobId.of(gcsBucket, pathPrefix + fileName);
        final Map<String, String> metaDataMap = new HashMap<>();
        metaDataMap.put("topic", topicName);
        metaDataMap.put("uuid", fileName);
        final BlobInfo objectInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").setMetadata(metaDataMap).build();
        try {
            final Blob objectCreated = gcsStore.create(objectInfo, topicMessagesMap.get(topicName).getBytes());
        } catch (JsonProcessingException jpe) {
            log.error("Exception::Failed to write to GCS: {} records size: {}", jpe, errorRecords.size());
            throw new BQErrorHandlerException(jpe.getMessage());
        }
    });
    log.info("Pushing {} records to GCS success?: {}", errorRecords.size(), true);
}
 

/** Example of how to release event-based hold for a blob */
public Blob releaseEventBasedHold(String bucketName, String blobName) throws StorageException {
  // [START storage_release_event_based_hold]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  // The name of a blob, e.g. "my-blob"
  // String blobName = "my-blob";

  BlobId blobId = BlobId.of(bucketName, blobName);
  Blob blob = storage.update(BlobInfo.newBuilder(blobId).setEventBasedHold(false).build());

  System.out.println("Event-based hold was released for " + blobName);
  // [END storage_release_event_based_hold]
  return blob;
}
 

@Test
public void testFailureOnException() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    runner.enqueue("testdata");

    when(storage.delete(any(BlobId.class))).thenThrow(new StorageException(1, "Test Exception"));

    runner.run();

    runner.assertPenalizeCount(1);
    runner.assertAllFlowFilesTransferred(DeleteGCSObject.REL_FAILURE);
    runner.assertTransferCount(DeleteGCSObject.REL_FAILURE, 1);
}
 

@Override
public FreshenResult freshenRemoteObject(final RemoteObjectReference object) {
    final BlobId blobId = ((GCPRemoteObjectReference) object).blobId;

    try {
        storage.copy(new Storage.CopyRequest.Builder()
                         .setSource(blobId)
                         .setTarget(BlobInfo.newBuilder(blobId).build(), Storage.BlobTargetOption.predefinedAcl(BUCKET_OWNER_FULL_CONTROL))
                         .build());

        return FreshenResult.FRESHENED;
    } catch (final StorageException e) {
        if (e.getCode() != 404) {
            throw e;
        }

        return FreshenResult.UPLOAD_REQUIRED;
    }
}
 

@Override
Tuple<BlobId, Map<String, String>> parse(String... args) {
  if (args.length < 2) {
    throw new IllegalArgumentException();
  }
  BlobId blobId = BlobId.of(args[0], args[1]);
  Map<String, String> metadata = new HashMap<>();
  for (int i = 2; i < args.length; i++) {
    int idx = args[i].indexOf('=');
    if (idx < 0) {
      metadata.put(args[i], "");
    } else {
      metadata.put(args[i].substring(0, idx), args[i].substring(idx + 1));
    }
  }
  return Tuple.of(blobId, metadata);
}
 

/** Example of deleting the ACL entry for an entity on a blob. */
// [TARGET deleteAcl(BlobId, Entity)]
// [VARIABLE "my_unique_bucket"]
// [VARIABLE "my_blob_name"]
// [VARIABLE 42]
public boolean deleteBlobAcl(String bucketName, String blobName, long blobGeneration) {
  // [START deleteBlobAcl]
  BlobId blobId = BlobId.of(bucketName, blobName, blobGeneration);
  boolean deleted = storage.deleteAcl(blobId, User.ofAllAuthenticatedUsers());
  if (deleted) {
    // the acl entry was deleted
  } else {
    // the acl entry was not found
  }
  // [END deleteBlobAcl]
  return deleted;
}
 

@Test
public void testStorageExceptionOnFetch() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    when(storage.get(any(BlobId.class))).thenThrow(new StorageException(400, "test-exception"));
    when(storage.reader(any(BlobId.class), any(Storage.BlobSourceOption.class))).thenReturn(new MockReadChannel(CONTENT));

    runner.enqueue("");

    runner.run();

    runner.assertAllFlowFilesTransferred(FetchGCSObject.REL_FAILURE);
    runner.assertTransferCount(FetchGCSObject.REL_FAILURE, 1);
}
 

/** Example of uploading a blob encrypted service side with a Cloud KMS key. */
public Blob createKmsEncrpytedBlob(String bucketName, String blobName, String kmsKeyName) {
  // [START storage_upload_with_kms_key]
  byte[] data = "Hello, World!".getBytes(UTF_8);

  // The name of the existing bucket to set a default KMS key for, e.g. "my-bucket"
  // String bucketName = "my-bucket"

  // The name of the KMS-key to use as a default
  // Key names are provided in the following format:
  // 'projects/<PROJECT>/locations/<LOCATION>/keyRings/<RING_NAME>/cryptoKeys/<KEY_NAME>'
  // String kmsKeyName = ""

  BlobId blobId = BlobId.of(bucketName, blobName);
  BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
  Blob blob = storage.create(blobInfo, data, BlobTargetOption.kmsKeyName(kmsKeyName));
  // [END storage_upload_with_kms_key]
  return blob;
}
 

@Override
Tuple<BlobId, Acl> parse(String... args) {
  if (args.length >= 3) {
    BlobId blob;
    int nextArg;
    if (args.length == 3) {
      blob = BlobId.of(args[0], "");
      nextArg = 1;
    } else if (args.length == 4) {
      blob = BlobId.of(args[0], args[1]);
      nextArg = 2;
    } else {
      throw new IllegalArgumentException("Too many arguments.");
    }
    String group = args[nextArg++];
    Acl.Role role = Acl.Role.valueOf(args[nextArg]);
    return Tuple.of(blob, Acl.of(new Acl.Group(group), role));
  }
  throw new IllegalArgumentException("Missing required bucket, groupEmail or role arguments.");
}
 

@Test
public void testFailureOnException() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    runner.enqueue("testdata");

    when(storage.delete(any(BlobId.class))).thenThrow(new StorageException(1, "Test Exception"));

    runner.run();

    runner.assertPenalizeCount(1);
    runner.assertAllFlowFilesTransferred(DeleteGCSObject.REL_FAILURE);
    runner.assertTransferCount(DeleteGCSObject.REL_FAILURE, 1);
}
 

/** Example of how to set event-based hold for a blob */
public Blob setEventBasedHold(String bucketName, String blobName) throws StorageException {
  // [START storage_set_event_based_hold]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  // The name of a blob, e.g. "my-blob"
  // String blobName = "my-blob";

  BlobId blobId = BlobId.of(bucketName, blobName);
  Blob blob = storage.update(BlobInfo.newBuilder(blobId).setEventBasedHold(true).build());

  System.out.println("Event-based hold was set for " + blobName);
  // [END storage_set_event_based_hold]
  return blob;
}
 

@Test
public void testDeleteWithValidArguments() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());
    addRequiredPropertiesToRunner(runner);

    runner.assertValid();

    runner.enqueue("testdata");

    runner.run();

    verify(storage).delete(eq(BlobId.of(BUCKET, KEY, GENERATION)));

    runner.assertAllFlowFilesTransferred(DeleteGCSObject.REL_SUCCESS);
    runner.assertTransferCount(DeleteGCSObject.REL_SUCCESS, 1);
}
 

private Optional<BigQuery.BigQueryErrors> load() {
  final Blob blob = mock(Blob.class);
  final String name = "OUTPUT_TABLE=dataset.table/x";
  when(blob.getName()).thenReturn(name);
  when(blob.getBlobId()).thenReturn(BlobId.of("bucket", name));
  when(blob.getSize()).thenReturn(1L);
  try {
    new BigQuery.Load(bigQuery, storage, 0, 0, Duration.ZERO, ForkJoinPool.commonPool(),
        BigQuery.Load.Delete.onSuccess).apply(blob).join();
  } catch (CompletionException e) {
    if (e.getCause() instanceof BigQuery.BigQueryErrors) {
      return Optional.of((BigQuery.BigQueryErrors) e.getCause());
    }
  }
  return Optional.empty();
}
 

public static void uploadObject(
    String projectId, String bucketName, String objectName, String filePath) throws IOException {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // The ID of your GCS object
  // String objectName = "your-object-name";

  // The path to your file to upload
  // String filePath = "path/to/your/file"

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  BlobId blobId = BlobId.of(bucketName, objectName);
  BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
  storage.create(blobInfo, Files.readAllBytes(Paths.get(filePath)));

  System.out.println(
      "File " + filePath + " uploaded to bucket " + bucketName + " as " + objectName);
}
 

/** Example of reading a blob's content through a reader. */
// [TARGET reader(BlobId, BlobSourceOption...)]
// [VARIABLE "my_unique_bucket"]
// [VARIABLE "my_blob_name"]
public void readerFromId(String bucketName, String blobName) throws IOException {
  // [START readerFromId]
  BlobId blobId = BlobId.of(bucketName, blobName);
  try (ReadChannel reader = storage.reader(blobId)) {
    ByteBuffer bytes = ByteBuffer.allocate(64 * 1024);
    while (reader.read(bytes) > 0) {
      bytes.flip();
      // do something with bytes
      bytes.clear();
    }
  }
  // [END readerFromId]
}
 

@Test
public void testFilePropagatedToLocalDirectory() {
	BlobId blobId = BlobId.of(this.cloudInputBucket, TEST_FILE_NAME);
	BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
	this.storage.create(blobInfo, "Hello World!".getBytes(StandardCharsets.UTF_8));

	Awaitility.await().atMost(15, TimeUnit.SECONDS)
			.untilAsserted(() -> {
				Path outputFile = Paths.get(this.outputFolder + "/" + TEST_FILE_NAME);
				assertThat(Files.exists(outputFile)).isTrue();
				assertThat(Files.isRegularFile(outputFile)).isTrue();

				String firstLine = Files.lines(outputFile).findFirst().get();
				assertThat(firstLine).isEqualTo("Hello World!");

				List<String> blobNamesInOutputBucket = new ArrayList<>();
				this.storage.list(this.cloudOutputBucket).iterateAll()
						.forEach((b) -> blobNamesInOutputBucket.add(b.getName()));

				assertThat(blobNamesInOutputBucket).contains(TEST_FILE_NAME);
			});
}
 

@Override
Tuple<BlobId, Acl> parse(String... args) {
  if (args.length >= 3) {
    BlobId blob;
    int nextArg;
    if (args.length == 3) {
      blob = BlobId.of(args[0], "");
      nextArg = 1;
    } else if (args.length == 4) {
      blob = BlobId.of(args[0], args[1]);
      nextArg = 2;
    } else {
      throw new IllegalArgumentException("Too many arguments.");
    }
    String domain = args[nextArg++];
    Acl.Role role = Acl.Role.valueOf(args[nextArg]);
    return Tuple.of(blob, Acl.of(new Acl.Domain(domain), role));
  }
  throw new IllegalArgumentException("Missing required bucket, domain or role arguments.");
}
 

/** Example of copying a blob in chunks. */
// [TARGET copy(CopyRequest)]
// [VARIABLE "my_unique_bucket"]
// [VARIABLE "my_blob_name"]
// [VARIABLE "copy_blob_name"]
public Blob copyBlobInChunks(String bucketName, String blobName, String copyBlobName) {
  // [START copyBlobInChunks]
  CopyRequest request =
      CopyRequest.newBuilder()
          .setSource(BlobId.of(bucketName, blobName))
          .setTarget(BlobId.of(bucketName, copyBlobName))
          .build();
  CopyWriter copyWriter = storage.copy(request);
  while (!copyWriter.isDone()) {
    copyWriter.copyChunk();
  }
  Blob blob = copyWriter.getResult();
  // [END copyBlobInChunks]
  return blob;
}
 

@Override
public Note get(String noteId, String notePath, AuthenticationInfo subject) throws IOException {
  BlobId blobId = makeBlobId(noteId, notePath);
  byte[] contents;
  try {
    contents = storage.readAllBytes(blobId);
  } catch (StorageException se) {
    throw new IOException("Could not read " + blobId.toString() + ": " + se.getMessage(), se);
  }

  try {
    return Note.fromJson(new String(contents, encoding));
  } catch (JsonParseException jpe) {
    throw new IOException(
        "Could note parse as json " + blobId.toString() + jpe.getMessage(), jpe);
  }
}
 

/**
 * Example of deleting a blob, only if its generation matches a value, otherwise a {@link
 * StorageException} is thrown.
 */
// [TARGET delete(BlobId, BlobSourceOption...)]
// [VARIABLE "my_unique_bucket"]
// [VARIABLE "my_blob_name"]
// [VARIABLE 42]
public boolean deleteBlobFromIdWithGeneration(
    String bucketName, String blobName, long blobGeneration) {
  // [START deleteBlobFromIdWithGeneration]
  BlobId blobId = BlobId.of(bucketName, blobName);
  boolean deleted = storage.delete(blobId, BlobSourceOption.generationMatch(blobGeneration));
  if (deleted) {
    // the blob was deleted
  } else {
    // the blob was not found
  }
  // [END deleteBlobFromIdWithGeneration]
  return deleted;
}
 

@Override
public Note get(String noteId, AuthenticationInfo subject) throws IOException {
  BlobId blobId = makeBlobId(noteId);
  byte[] contents;
  try {
    contents = storage.readAllBytes(blobId);
  } catch (StorageException se) {
    throw new IOException("Could not read " + blobId.toString() + ": " + se.getMessage(), se);
  }

  try {
    return Note.fromJson(new String(contents, encoding));
  } catch (JsonParseException jpe) {
    throw new IOException(
        "Could note parse as json " + blobId.toString() + jpe.getMessage(), jpe);
  }
}
 

/**
 * Archive the blob
 * @param blobId blobId
 */
private void handleArchive(BlobId blobId) {
  String destinationPath = getDestinationPath(blobId, gcsOriginErrorConfig.errorPrefix);
  switch (gcsOriginErrorConfig.archivingOption) {
    case COPY_TO_BUCKET:
      copy(blobId, gcsOriginErrorConfig.errorBucket, destinationPath, false);
      break;
    case MOVE_TO_BUCKET:
      copy(blobId, gcsOriginErrorConfig.errorBucket, destinationPath, true);
      break;
    case COPY_TO_PREFIX:
      copy(blobId, blobId.getBucket(), destinationPath, false);
      break;
    case MOVE_TO_PREFIX:
      copy(blobId, blobId.getBucket(), destinationPath, true);
      break;
  }
}
 

@Before
public void setUp() {
  // Create the product set csv file locally and upload it to GCS
  // This is so that there is a unique product set ID for all java version tests.
  Storage storage = StorageOptions.newBuilder().setProjectId(PROJECT_ID).build().getService();
  BlobId blobId = BlobId.of(PROJECT_ID, FILEPATH);
  BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
  String csvContents =
      "\"gs://cloud-samples-data/vision/product_search/shoes_1.jpg\","
          + String.format("\"%s\",", IMAGE_URI_1)
          + String.format("\"%s\",", PRODUCT_SET_ID)
          + String.format("\"%s\",", PRODUCT_ID_1)
          + "\"apparel\",,\"style=womens\",\"0.1,0.1,0.9,0.1,0.9,0.9,0.1,0.9\"";
  blob = storage.create(blobInfo, csvContents.getBytes());

  bout = new ByteArrayOutputStream();
  out = new PrintStream(bout);
  System.setOut(out);
}
 

public static void downloadRequesterPaysObject(
    String projectId, String bucketName, String objectName, Path destFilePath) {
  // The project ID to bill
  // String projectId = "my-billable-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // The ID of your GCS object
  // String objectName = "your-object-name";

  // The path to which the file should be downloaded
  // Path destFilePath = Paths.get("/local/path/to/file.txt");

  Storage storage = StorageOptions.getDefaultInstance().getService();
  Blob blob = storage.get(BlobId.of(bucketName, objectName));
  blob.downloadTo(destFilePath, Blob.BlobSourceOption.userProject(projectId));

  System.out.println(
      "Object " + objectName + " downloaded to " + destFilePath + " and billed to " + projectId);
}
 

public static void changeObjectStorageClass(
    String projectId, String bucketName, String objectName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // The ID of your GCS object
  // String objectName = "your-object-name";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  BlobId blobId = BlobId.of(bucketName, objectName);

  // See the StorageClass documentation for other valid storage classes:
  // https://googleapis.dev/java/google-cloud-clients/latest/com/google/cloud/storage/StorageClass.html
  StorageClass storageClass = StorageClass.COLDLINE;

  // You can't change an object's storage class directly, the only way is to rewrite the object
  // with the
  // desired storage class
  Storage.CopyRequest request =
      Storage.CopyRequest.newBuilder()
          .setSource(blobId)
          .setTarget(BlobInfo.newBuilder(blobId).setStorageClass(storageClass).build())
          .build();
  Blob updatedBlob = storage.copy(request).getResult();

  System.out.println(
      "Object "
          + objectName
          + " in bucket "
          + bucketName
          + " had its storage class set to "
          + updatedBlob.getStorageClass().name());
}
 

/** Example of getting information on several blobs using a single batch request. */
// [TARGET get(Iterable)]
// [VARIABLE "my_unique_bucket"]
// [VARIABLE "my_blob_name1"]
// [VARIABLE "my_blob_name2"]
public List<Blob> batchGetIterable(String bucketName, String blobName1, String blobName2) {
  // [START batchGetIterable]
  List<BlobId> blobIds = new LinkedList<>();
  blobIds.add(BlobId.of(bucketName, blobName1));
  blobIds.add(BlobId.of(bucketName, blobName2));
  List<Blob> blobs = storage.get(blobIds);
  // [END batchGetIterable]
  return blobs;
}
 

@Override
public String downloadFileToString(final Path localFile, final RemoteObjectReference objectReference) throws Exception {
    final BlobId blobId = ((GCPRemoteObjectReference) objectReference).blobId;
    Files.createDirectories(localFile.getParent());

    try (final ReadChannel inputChannel = storage.reader(blobId)) {
        try (final InputStreamReader isr = new InputStreamReader(Channels.newInputStream(inputChannel))) {
            return CharStreams.toString(isr);
        }
    }
}
 

@Test
public void testAclOwnerUser() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    final Blob blob = mock(Blob.class);

    final Acl.User mockUser = mock(Acl.User.class);
    when(mockUser.getEmail()).thenReturn(OWNER_USER_EMAIL);
    when(blob.getOwner()).thenReturn(mockUser);

    when(storage.get(any(BlobId.class))).thenReturn(blob);
    when(storage.reader(any(BlobId.class), any(Storage.BlobSourceOption.class))).thenReturn(new MockReadChannel(CONTENT));

    runner.enqueue("");

    runner.run();

    verify(storage).get(any(BlobId.class));
    verify(storage).reader(any(BlobId.class), any(Storage.BlobSourceOption.class));

    runner.assertAllFlowFilesTransferred(FetchGCSObject.REL_SUCCESS);
    runner.assertTransferCount(FetchGCSObject.REL_SUCCESS, 1);
    final MockFlowFile flowFile = runner.getFlowFilesForRelationship(FetchGCSObject.REL_SUCCESS).get(0);

    assertEquals(
            OWNER_USER_EMAIL,
            flowFile.getAttribute(OWNER_ATTR)
    );

    assertEquals(
            "user",
            flowFile.getAttribute(OWNER_TYPE_ATTR)
    );
}
 

/** Example of getting the ACL entry for an entity on a blob. */
// [TARGET getAcl(BlobId, Entity)]
// [VARIABLE "my_unique_bucket"]
// [VARIABLE "my_blob_name"]
// [VARIABLE 42]
public Acl getBlobAcl(String bucketName, String blobName, long blobGeneration) {
  // [START getBlobAcl]
  BlobId blobId = BlobId.of(bucketName, blobName, blobGeneration);
  Acl acl = storage.getAcl(blobId, User.ofAllAuthenticatedUsers());
  // [END getBlobAcl]
  return acl;
}
 

@Test
public void testGetInputStreamOnNullBlob() throws Exception {
	this.expectedEx.expect(FileNotFoundException.class);
	this.expectedEx.expectMessage("The blob was not found: gs://test-spring/test");
	String location = "gs://test-spring/test";
	Storage storage = mock(Storage.class);
	when(storage.get(BlobId.of("test-spring", "test"))).thenReturn(null);

	GoogleStorageResource resource = new GoogleStorageResource(storage, location,
			false);
	resource.getInputStream();
}