com.google.cloud.storage.Storage.BlobListOption Java Exaples

Source File: ITTranslateSnippetsBeta.java From google-cloud-java with Apache License 2.0

6 votes

@Test
public void test1_testBatchTranslateText() {

  BatchTranslateResponse response =
      TranslateSnippetsBeta.batchTranslateText(
          projectId,
          "us-central1",
          "gs://cloud-samples-data/translation/text.txt",
          "gs://" + projectId + "/BATCH_TRANSLATION_OUTPUT/");
  assertEquals(13, response.getTotalCharacters());
  assertEquals(13, response.getTranslatedCharacters());
  Storage storage = StorageOptions.getDefaultInstance().getService();

  Page<Blob> blobs =
      storage.list(
          projectId,
          BlobListOption.currentDirectory(),
          BlobListOption.prefix("BATCH_TRANSLATION_OUTPUT/"));

  deleteDirectory(storage, blobs);
}

Source File: GCSStorage.java From digdag with Apache License 2.0

6 votes

@Override
public void list(String objectPrefix, FileListing callback)
{
    checkArgument(objectPrefix != null, "objectPrefix is null");

    String errorMessage = "listing files on bucket " + bucket + " prefix " + objectPrefix;
    Page<Blob> blobs = getWithRetry(errorMessage, () ->
            storage.list(bucket, BlobListOption.prefix(objectPrefix))
    );

    List<StorageObjectSummary> objectSummaryList = new ArrayList<>();
    for (Blob blob : blobs.iterateAll()) {
        objectSummaryList.add(
                StorageObjectSummary.builder()
                        .key(blob.getName())
                        .contentLength(blob.getSize())
                        .lastModified(convertToInstant(blob))
                        .build()
        );
    }
    callback.accept(objectSummaryList);
}

Source File: DocumentOcrTemplate.java From spring-cloud-gcp with Apache License 2.0

6 votes

/**
 * Parses the OCR output files who have the specified {@code jsonFilesetPrefix}. This
 * method assumes that all of the OCR output files with the prefix are a part of the same
 * document.
 *
 * @param jsonOutputFilePathPrefix the folder location containing all of the JSON files of
 *     OCR output
 * @return A {@link DocumentOcrResultSet} describing the OCR content of a document
 */
public DocumentOcrResultSet readOcrOutputFileSet(GoogleStorageLocation jsonOutputFilePathPrefix) {
	String nonNullPrefix = (jsonOutputFilePathPrefix.getBlobName() == null)
			? ""
			: jsonOutputFilePathPrefix.getBlobName();

	Page<Blob> blobsInFolder = this.storage.list(
			jsonOutputFilePathPrefix.getBucketName(),
			BlobListOption.currentDirectory(),
			BlobListOption.prefix(nonNullPrefix));

	List<Blob> blobPages =
			StreamSupport.stream(blobsInFolder.getValues().spliterator(), false)
					.filter(blob -> blob.getContentType().equals("application/octet-stream"))
					.collect(Collectors.toList());

	return new DocumentOcrResultSet(blobPages);
}

Source File: SnowflakeGCSClient.java From snowflake-jdbc with Apache License 2.0

6 votes

/**
 * listObjects gets all the objects in a path
 *
 * @param remoteStorageLocation bucket name
 * @param prefix                Path
 * @return
 * @throws StorageProviderException
 */
@Override
public StorageObjectSummaryCollection listObjects(String remoteStorageLocation,
                                                  String prefix) throws StorageProviderException
{
  try
  {
    Page<Blob> blobs = this.gcsClient.list(remoteStorageLocation, BlobListOption.prefix(prefix));
    return new StorageObjectSummaryCollection(blobs);
  }
  catch (Exception e)
  {
    logger.debug("Failed to list objects");
    throw new StorageProviderException(e);
  }
}

Source File: StreamingAnnotationToStorageIT.java From java-docs-samples with Apache License 2.0

6 votes

@Test
public void testStreamingAnnotationToStorage() {
  String gcsUri = String.format("gs://%s/%s", PROJECT_ID, OUTPUT_PREFIX);
  StreamingAnnotationToStorage.streamingAnnotationToStorage("resources/cat.mp4", gcsUri);
  String got = bout.toString();

  assertThat(got).contains(String.format("Storage Uri: %s", gcsUri));

  Storage storage = StorageOptions.getDefaultInstance().getService();

  Page<Blob> blobs =
      storage.list(
          PROJECT_ID,
          BlobListOption.currentDirectory(),
          BlobListOption.prefix(OUTPUT_PREFIX + "/"));

  deleteDirectory(storage, blobs);
}

Source File: DetectIT.java From java-docs-samples with Apache License 2.0

6 votes

@Test
public void testDetectDocumentsGcs() throws Exception {
  // Act
  Detect.detectDocumentsGcs(
      "gs://" + ASSET_BUCKET + "/vision/document/custom_0773375000.pdf",
      "gs://" + OUTPUT_BUCKET + "/" + OUTPUT_PREFIX + "/");

  // Assert
  String got = bout.toString();

  assertThat(got).contains("OIL, GAS AND MINERAL LEASE");

  Storage storage = StorageOptions.getDefaultInstance().getService();

  Page<Blob> blobs =
      storage.list(
          OUTPUT_BUCKET,
          BlobListOption.currentDirectory(),
          BlobListOption.prefix(OUTPUT_PREFIX + "/"));
  for (Blob blob : blobs.iterateAll()) {
    blob.delete();
  }
}

Source File: DetectIT.java From java-docs-samples with Apache License 2.0

6 votes

@Test
public void testAsyncBatchAnnotateImagesGcs() throws Exception {
  // Act
  AsyncBatchAnnotateImagesGcs.asyncBatchAnnotateImagesGcs(
      "gs://cloud-samples-data/vision/label/wakeupcat.jpg",
      "gs://" + OUTPUT_BUCKET + "/" + OUTPUT_PREFIX + "/");

  // Assert
  String got = bout.toString();
  assertThat(got).contains("red:");

  Storage storage = StorageOptions.getDefaultInstance().getService();

  Page<Blob> blobs = storage.list(OUTPUT_BUCKET, BlobListOption.currentDirectory(),
      BlobListOption.prefix(OUTPUT_PREFIX + "/"));
  for (Blob blob : blobs.iterateAll()) {
    blob.delete();
  }
}

Source File: QuickStartIT.java From java-docs-samples with Apache License 2.0

6 votes

private static final void deleteObjects() {
  Storage storage = StorageOptions.getDefaultInstance().getService();
  for (BlobInfo info :
      storage
          .list(
              bucketName,
              BlobListOption.versions(true),
              BlobListOption.currentDirectory(),
              BlobListOption.prefix(path + "/"))
          .getValues()) {
    storage.delete(info.getBlobId());
  }
}

Source File: GoogleCloudBlobStoreMetricsStore.java From nexus-blobstore-google-cloud with Eclipse Public License 1.0

5 votes

protected Stream<GoogleCloudPropertiesFile> backingFiles() {
  if (bucket == null) {
    return Stream.empty();
  } else {
    return stream(bucket.list(BlobListOption.prefix(nodeAccess.getId())).iterateAll())
        .filter(b -> b.getName().endsWith(METRICS_FILENAME))
        .map(blob -> new GoogleCloudPropertiesFile(bucket, blob.getName()));
  }
}

Source File: ITTranslateSnippetsBeta.java From google-cloud-java with Apache License 2.0

5 votes

private void deleteDirectory(Storage storage, Page<Blob> blobs) {
  for (Blob blob : blobs.iterateAll()) {
    System.out.println(blob.getBlobId());
    if (!blob.delete()) {
      Page<Blob> subBlobs =
          storage.list(
              projectId,
              BlobListOption.currentDirectory(),
              BlobListOption.prefix(blob.getName()));

      deleteDirectory(storage, subBlobs);
    }
  }
}

Source File: StreamingAnnotationToStorageIT.java From java-docs-samples with Apache License 2.0

5 votes

private void deleteDirectory(Storage storage, Page<Blob> blobs) {
  for (Blob blob : blobs.iterateAll()) {
    System.out.println(blob.getName());
    if (!blob.delete()) {
      Page<Blob> subBlobs =
          storage.list(
              PROJECT_ID,
              BlobListOption.currentDirectory(),
              BlobListOption.prefix(blob.getName()));

      deleteDirectory(storage, subBlobs);
    }
  }
}

Source File: OldGCSNotebookRepo.java From zeppelin with Apache License 2.0

5 votes

@Override
public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
  try {
    List<OldNoteInfo> infos = new ArrayList<>();
    Iterable<Blob> blobsUnderDir;
    if (basePath.isPresent()) {
      blobsUnderDir = storage
        .list(bucketName, BlobListOption.prefix(this.basePath.get() + "/"))
        .iterateAll();
    } else {
      blobsUnderDir = storage
        .list(bucketName)
        .iterateAll();
    }
    for (Blob b : blobsUnderDir) {
      Matcher matcher = noteNamePattern.matcher(b.getName());
      if (matcher.matches()) {
        // Callers only use the id field, so do not fetch each note
        // This matches the implementation in FileSystemNoteRepo#list
        infos.add(new OldNoteInfo(matcher.group(1), "", null));
      }
    }
    return infos;
  } catch (StorageException se) {
    throw new IOException("Could not list GCS directory: " + se.getMessage(), se);
  }
}

Source File: GCSNotebookRepo.java From zeppelin with Apache License 2.0

5 votes

@Override
public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException {
  try {
    Map<String, NoteInfo> infos = new HashMap<>();
    Iterable<Blob> blobsUnderDir;
    if (basePath.isPresent()) {
      blobsUnderDir = storage
        .list(bucketName, BlobListOption.prefix(this.basePath.get() + "/"))
        .iterateAll();
    } else {
      blobsUnderDir = storage
        .list(bucketName)
        .iterateAll();
    }
    for (Blob b : blobsUnderDir) {
      Matcher matcher = notePathPattern.matcher(b.getName());
      if (matcher.matches()) {
        // Callers only use the id field, so do not fetch each note
        // This matches the implementation in FileSystemNoteRepo#list
        String noteFileName = matcher.group(1);
        try {
          String noteId = getNoteId(noteFileName);
          String notePath = getNotePath("", noteFileName);
          infos.put(noteId, new NoteInfo(noteId, notePath));
        } catch (IOException e) {
          LOGGER.warn(e.getMessage());
        }
      }
    }
    return infos;
  } catch (StorageException se) {
    throw new IOException("Could not list GCS directory: " + se.getMessage(), se);
  }
}

Source File: GCPRestorer.java From cassandra-backup with Apache License 2.0

5 votes

private Page<Blob> list(final String bucket, final String pathPrefix) {
    return storage.list(bucket, BlobListOption.prefix(request.storageLocation.clusterId
                                                          + "/" + request.storageLocation.datacenterId
                                                          + "/" + request.storageLocation.nodeId
                                                          + "/" + pathPrefix + "/"),
                        BlobListOption.currentDirectory());
}

Source File: StorageIntegrationTest.java From gcp-ingestion with Mozilla Public License 2.0

4 votes

private void downloadOutputFiles(String prefix) {
  Page<Blob> blobs = storage.list(bucket, BlobListOption.prefix(prefix));
  for (Blob blob : blobs.iterateAll()) {
    blob.downloadTo(Paths.get(tempFolder.getRoot().getPath(), blob.getName()));
  }
}

Source File: GoogleCloudBlobStore.java From nexus-blobstore-google-cloud with Eclipse Public License 1.0

4 votes

Stream<BlobInfo> blobStream(final String path) {
  return stream(bucket.list(BlobListOption.prefix(path)).iterateAll()).map(c -> c);
}

Source File: ITBlobSnippets.java From google-cloud-java with Apache License 2.0

4 votes

@After
public void after() {
  for (BlobInfo info : storage.list(BUCKET, BlobListOption.versions(true)).getValues()) {
    storage.delete(info.getBlobId());
  }
}

com.google.cloud.storage.Storage.BlobListOption Java Examples