com.google.api.gax.paging.Page Java Examples

private static String getLogEntriesAsString(String startTimestamp) {
  // Construct Stackdriver logging filter
  // See this page for more info: https://cloud.google.com/logging/docs/view/advanced-queries
  String filter = "resource.type=\"cloud_function\""
      + " AND severity=INFO"
      + " AND resource.labels.function_name=" + FUNCTION_DEPLOYED_NAME
      + String.format(" AND timestamp>=\"%s\"", startTimestamp);

  // Get Stackdriver logging entries
  Page<LogEntry> logEntries =
      loggingClient.listLogEntries(
          Logging.EntryListOption.filter(filter),
          Logging.EntryListOption.sortOrder(
              Logging.SortingField.TIMESTAMP, Logging.SortingOrder.DESCENDING)
      );

  // Serialize Stackdriver logging entries + collect them into a single string
  String logsConcat = StreamSupport.stream(logEntries.getValues().spliterator(), false)
      .map((x) -> x.toString())
      .collect(Collectors.joining("%n"));

  return logsConcat;
}
 

/**
 * listObjects gets all the objects in a path
 *
 * @param remoteStorageLocation bucket name
 * @param prefix                Path
 * @return
 * @throws StorageProviderException
 */
@Override
public StorageObjectSummaryCollection listObjects(String remoteStorageLocation,
                                                  String prefix) throws StorageProviderException
{
  try
  {
    Page<Blob> blobs = this.gcsClient.list(remoteStorageLocation, BlobListOption.prefix(prefix));
    return new StorageObjectSummaryCollection(blobs);
  }
  catch (Exception e)
  {
    logger.debug("Failed to list objects");
    throw new StorageProviderException(e);
  }
}
 

@Override
public Path downloadFileToDir(final Path destinationDir, final Path remotePrefix, final Predicate<String> keyFilter) throws Exception {

    final Page<Blob> blobs = list(request.storageLocation.bucket, remotePrefix.toString());
    final List<Blob> blobItems = new ArrayList<>();

    for (final Blob blob : blobs.iterateAll()) {
        if (keyFilter.test(blob.getName())) {
            blobItems.add(blob);
        }
    }

    if (blobItems.size() != 1) {
        throw new IllegalStateException(format("There is not one key which satisfies key filter: %s", blobItems.toString()));
    }

    final String blobItemPath = blobItems.get(0).getName();
    final String fileName = blobItemPath.split("/")[blobItemPath.split("/").length - 1];

    final Path destination = destinationDir.resolve(fileName);

    downloadFile(destination, objectKeyToRemoteReference(remotePrefix.resolve(fileName)));

    return destination;
}
 

@Test
public void test1_testBatchTranslateText() {

  BatchTranslateResponse response =
      TranslateSnippetsBeta.batchTranslateText(
          projectId,
          "us-central1",
          "gs://cloud-samples-data/translation/text.txt",
          "gs://" + projectId + "/BATCH_TRANSLATION_OUTPUT/");
  assertEquals(13, response.getTotalCharacters());
  assertEquals(13, response.getTranslatedCharacters());
  Storage storage = StorageOptions.getDefaultInstance().getService();

  Page<Blob> blobs =
      storage.list(
          projectId,
          BlobListOption.currentDirectory(),
          BlobListOption.prefix("BATCH_TRANSLATION_OUTPUT/"));

  deleteDirectory(storage, blobs);
}
 

/**
 * Parses the OCR output files who have the specified {@code jsonFilesetPrefix}. This
 * method assumes that all of the OCR output files with the prefix are a part of the same
 * document.
 *
 * @param jsonOutputFilePathPrefix the folder location containing all of the JSON files of
 *     OCR output
 * @return A {@link DocumentOcrResultSet} describing the OCR content of a document
 */
public DocumentOcrResultSet readOcrOutputFileSet(GoogleStorageLocation jsonOutputFilePathPrefix) {
	String nonNullPrefix = (jsonOutputFilePathPrefix.getBlobName() == null)
			? ""
			: jsonOutputFilePathPrefix.getBlobName();

	Page<Blob> blobsInFolder = this.storage.list(
			jsonOutputFilePathPrefix.getBucketName(),
			BlobListOption.currentDirectory(),
			BlobListOption.prefix(nonNullPrefix));

	List<Blob> blobPages =
			StreamSupport.stream(blobsInFolder.getValues().spliterator(), false)
					.filter(blob -> blob.getContentType().equals("application/octet-stream"))
					.collect(Collectors.toList());

	return new DocumentOcrResultSet(blobPages);
}
 

@Test
public void testListHmacKeys() {
  // Create 2 HMAC keys
  storage.createHmacKey(
      ServiceAccount.of(HMAC_KEY_TEST_SERVICE_ACCOUNT),
      Storage.CreateHmacKeyOption.projectId(PROJECT_ID));
  storage.createHmacKey(
      ServiceAccount.of(HMAC_KEY_TEST_SERVICE_ACCOUNT),
      Storage.CreateHmacKeyOption.projectId(PROJECT_ID));

  Page<HmacKeyMetadata> page = storageSnippets.listHmacKeys(PROJECT_ID);

  int count = 0;
  for (HmacKeyMetadata metadata : page.iterateAll()) {
    if (metadata.getServiceAccount().getEmail().equals(HMAC_KEY_TEST_SERVICE_ACCOUNT)) {
      count++;
    }
  }

  assertEquals(2, count);
}
 

/**
 * Determines if a path is a directory that is not empty
 * @param uri The path under the gcs bucket
 * @return {@code true} if the path is a non-empty directory,
 *         {@code false} otherwise
 */
private boolean isEmptyDirectory(URI uri) throws IOException {
  if (!isDirectory(uri)) {
    return false;
  }
  String prefix = normalizeToDirectoryPrefix(uri);
  boolean isEmpty = true;
  Page<Blob> page;
  if (prefix.equals(DELIMITER)) {
    page = getBucket(uri).list();
  } else {
    page = getBucket(uri).list(Storage.BlobListOption.prefix(prefix));
  }
  for (Blob blob : page.iterateAll()) {
    if (blob.getName().equals(prefix)) {
      continue;
    } else {
      isEmpty = false;
      break;
    }
  }
  return isEmpty;
}
 

@Override
public String[] listFiles(URI fileUri, boolean recursive) throws IOException {
  try {
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    String prefix = normalizeToDirectoryPrefix(fileUri);
    Page<Blob> page;
    if (recursive) {
      page = storage.list(fileUri.getHost(), Storage.BlobListOption.prefix(prefix));
    } else {
      page = storage.list(fileUri.getHost(), Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory());
    }
    page.iterateAll()
        .forEach(blob -> {
          if (!blob.getName().equals(fileUri.getPath())) {
            builder.add(blob.getName());
          }
        });
    return builder.build().toArray(new String[0]);
  } catch (Throwable t) {
    throw new IOException(t);
  }
}
 

/** Expects an existing Stackdriver log name as an argument. */
public static void main(String... args) throws Exception {
  // [START logging_list_log_entries]
  // Instantiates a client
  LoggingOptions options = LoggingOptions.getDefaultInstance();

  String logName = args[0];

  try (Logging logging = options.getService()) {

    String logFilter = "logName=projects/" + options.getProjectId() + "/logs/" + logName;

    // List all log entries
    Page<LogEntry> entries = logging.listLogEntries(
        EntryListOption.filter(logFilter));
    do {
      for (LogEntry logEntry : entries.iterateAll()) {
        System.out.println(logEntry);
      }
      entries = entries.getNextPage();
    } while (entries != null);

  }
  // [END logging_list_log_entries]
}
 

public static void main(String... args) throws Exception {
  // Create a service object
  // Credentials are inferred from the environment
  try (Logging logging = LoggingOptions.getDefaultInstance().getService()) {

    // Create a metric
    MetricInfo metricInfo =
        MetricInfo.newBuilder("test-metric", "severity >= ERROR")
            .setDescription("Log entries with severity higher or equal to ERROR")
            .build();
    logging.create(metricInfo);

    // List metrics
    Page<Metric> metrics = logging.listMetrics();
    for (Metric metric : metrics.iterateAll()) {
      System.out.println(metric);
    }
  }
}
 

public static void main(String... args) throws Exception {
  // Create a service object
  // Credentials are inferred from the environment
  try (Logging logging = LoggingOptions.getDefaultInstance().getService()) {

    // Create a sink to back log entries to a BigQuery dataset
    SinkInfo sinkInfo =
        SinkInfo.newBuilder("test-sink", DatasetDestination.of("test-dataset"))
            .setFilter("severity >= ERROR")
            .build();
    logging.create(sinkInfo);

    // List sinks
    Page<Sink> sinks = logging.listSinks();
    for (Sink sink : sinks.iterateAll()) {
      System.out.println(sink);
    }
  }
}
 

public Page<HmacKeyMetadata> listHmacKeys(String projectId) throws StorageException {
  // [START storage_list_hmac_keys]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The ID of the project to which the service account belongs.
  // String projectId = "project-id";
  Page<HmacKeyMetadata> page = storage.listHmacKeys(ListHmacKeysOption.projectId(projectId));

  for (HmacKeyMetadata metadata : page.iterateAll()) {
    System.out.println("Service Account Email: " + metadata.getServiceAccount().getEmail());
    System.out.println("Access ID: " + metadata.getAccessId());
  }
  // [END storage_list_hmac_keys]
  return page;
}
 

private static String getLogEntriesAsString(String startTimestamp) {
  // Construct Stackdriver logging filter
  // See this page for more info: https://cloud.google.com/logging/docs/view/advanced-queries
  String filter = "resource.type=\"cloud_function\""
      + " AND severity=INFO"
      + " AND resource.labels.function_name=" + FUNCTION_DEPLOYED_NAME
      + String.format(" AND timestamp>=\"%s\"", startTimestamp);

  // Get Stackdriver logging entries
  Page<LogEntry> logEntries =
      loggingClient.listLogEntries(
          Logging.EntryListOption.filter(filter),
          Logging.EntryListOption.sortOrder(
              Logging.SortingField.TIMESTAMP, Logging.SortingOrder.DESCENDING)
      );

  // Serialize Stackdriver logging entries + collect them into a single string
  String logsConcat = StreamSupport.stream(logEntries.getValues().spliterator(), false)
      .map((x) -> x.toString())
      .collect(Collectors.joining("%n"));

  return logsConcat;
}
 

@Test
public void testAsyncBatchAnnotateImagesGcs() throws Exception {
  // Act
  AsyncBatchAnnotateImagesGcs.asyncBatchAnnotateImagesGcs(
      "gs://cloud-samples-data/vision/label/wakeupcat.jpg",
      "gs://" + OUTPUT_BUCKET + "/" + OUTPUT_PREFIX + "/");

  // Assert
  String got = bout.toString();
  assertThat(got).contains("red:");

  Storage storage = StorageOptions.getDefaultInstance().getService();

  Page<Blob> blobs = storage.list(OUTPUT_BUCKET, BlobListOption.currentDirectory(),
      BlobListOption.prefix(OUTPUT_PREFIX + "/"));
  for (Blob blob : blobs.iterateAll()) {
    blob.delete();
  }
}
 

static void listBackupOperations(InstanceAdminClient instanceAdminClient, DatabaseId databaseId) {
  Instance instance = instanceAdminClient.getInstance(databaseId.getInstanceId().getInstance());
  // Get create backup operations for the sample database.
  String filter =
      String.format(
          "(metadata.database:%s) AND "
              + "(metadata.@type:type.googleapis.com/"
              + "google.spanner.admin.database.v1.CreateBackupMetadata)",
          databaseId.getName());
  Page<Operation> operations = instance.listBackupOperations(Options.filter(filter));
  for (Operation op : operations.iterateAll()) {
    try {
      CreateBackupMetadata metadata = op.getMetadata().unpack(CreateBackupMetadata.class);
      System.out.println(
          String.format(
              "Backup %s on database %s pending: %d%% complete",
              metadata.getName(),
              metadata.getDatabase(),
              metadata.getProgress().getProgressPercent()));
    } catch (InvalidProtocolBufferException e) {
      // The returned operation does not contain CreateBackupMetadata.
      System.err.println(e.getMessage());
    }
  }
}
 

@Test
public void testDetectDocumentsGcs() throws Exception {
  // Act
  Detect.detectDocumentsGcs(
      "gs://" + ASSET_BUCKET + "/vision/document/custom_0773375000.pdf",
      "gs://" + OUTPUT_BUCKET + "/" + OUTPUT_PREFIX + "/");

  // Assert
  String got = bout.toString();

  assertThat(got).contains("OIL, GAS AND MINERAL LEASE");

  Storage storage = StorageOptions.getDefaultInstance().getService();

  Page<Blob> blobs =
      storage.list(
          OUTPUT_BUCKET,
          BlobListOption.currentDirectory(),
          BlobListOption.prefix(OUTPUT_PREFIX + "/"));
  for (Blob blob : blobs.iterateAll()) {
    blob.delete();
  }
}
 

@Test
public void testStreamingAnnotationToStorage() {
  String gcsUri = String.format("gs://%s/%s", PROJECT_ID, OUTPUT_PREFIX);
  StreamingAnnotationToStorage.streamingAnnotationToStorage("resources/cat.mp4", gcsUri);
  String got = bout.toString();

  assertThat(got).contains(String.format("Storage Uri: %s", gcsUri));

  Storage storage = StorageOptions.getDefaultInstance().getService();

  Page<Blob> blobs =
      storage.list(
          PROJECT_ID,
          BlobListOption.currentDirectory(),
          BlobListOption.prefix(OUTPUT_PREFIX + "/"));

  deleteDirectory(storage, blobs);
}
 

/** Example of asynchronously listing log entries for a specific log. */
// [TARGET listLogEntriesAsync(EntryListOption...)]
// [VARIABLE "logName=projects/my_project_id/logs/my_log_name"]
public Page<LogEntry> listLogEntriesAsync(String filter)
    throws ExecutionException, InterruptedException {
  // [START listLogEntriesAsync]
  Future<AsyncPage<LogEntry>> future =
      logging.listLogEntriesAsync(EntryListOption.filter(filter));
  // ...
  AsyncPage<LogEntry> entries = future.get();
  for (LogEntry entry : entries.iterateAll()) {
    // do something with the entry
  }
  // [END listLogEntriesAsync]
  return entries;
}
 

@Override
public void list(String objectPrefix, FileListing callback)
{
    checkArgument(objectPrefix != null, "objectPrefix is null");

    String errorMessage = "listing files on bucket " + bucket + " prefix " + objectPrefix;
    Page<Blob> blobs = getWithRetry(errorMessage, () ->
            storage.list(bucket, BlobListOption.prefix(objectPrefix))
    );

    List<StorageObjectSummary> objectSummaryList = new ArrayList<>();
    for (Blob blob : blobs.iterateAll()) {
        objectSummaryList.add(
                StorageObjectSummary.builder()
                        .key(blob.getName())
                        .contentLength(blob.getSize())
                        .lastModified(convertToInstant(blob))
                        .build()
        );
    }
    callback.accept(objectSummaryList);
}
 

/** Example of listing datasets in a project, specifying the page size. */
// [TARGET listDatasets(String, DatasetListOption...)]
// [VARIABLE "my_project_id"]
public Page<Dataset> listDatasets(String projectId) {
  // [START bigquery_list_datasets]
  // List datasets in a specified project
  Page<Dataset> datasets = bigquery.listDatasets(projectId, DatasetListOption.pageSize(100));
  for (Dataset dataset : datasets.iterateAll()) {
    // do something with the dataset
  }
  // [END bigquery_list_datasets]
  return datasets;
}
 

/** Example of listing the tables in a dataset, specifying the page size. */
// [TARGET listTables(String, TableListOption...)]
// [VARIABLE "my_dataset_name"]
public Page<Table> listTables(String datasetName) {
  // [START ]
  Page<Table> tables = bigquery.listTables(datasetName, TableListOption.pageSize(100));
  for (Table table : tables.iterateAll()) {
    // do something with the table
  }
  // [END ]
  return tables;
}
 

@Override
public void run(Compute compute, RegionId region) {
  Page<Subnetwork> subnetworkPage;
  if (region != null) {
    subnetworkPage = compute.listSubnetworks(region.getRegion());
  } else {
    subnetworkPage = compute.listSubnetworks();
  }
  for (Subnetwork subnetwork : subnetworkPage.iterateAll()) {
    System.out.println(subnetwork);
  }
}
 

/** Example of listing metrics, specifying the page size. */
// [TARGET listMetrics(ListOption...)]
public Page<Metric> listMetrics() {
  // [START listMetrics]
  Page<Metric> metrics = logging.listMetrics(ListOption.pageSize(100));
  for (Metric metric : metrics.iterateAll()) {
    // do something with the metric
  }
  // [END listMetrics]
  return metrics;
}
 

/** Example of listing sinks, specifying the page size. */
// [TARGET listSinks(ListOption...)]
public Page<Sink> listSinks() {
  // [START logging_list_sinks]
  Page<Sink> sinks = logging.listSinks(ListOption.pageSize(100));
  for (Sink sink : sinks.iterateAll()) {
    // do something with the sink
  }
  // [END logging_list_sinks]
  return sinks;
}
 

/** Example of listing jobs, specifying the page size. */
// [TARGET listJobs(JobListOption...)]
public Page<Job> listJobs() {
  // [START bigquery_list_jobs]
  Page<Job> jobs = bigquery.listJobs(JobListOption.pageSize(100));
  for (Job job : jobs.iterateAll()) {
    // do something with the job
  }
  // [END bigquery_list_jobs]
  return jobs;
}
 

/** Example of listing the tables in a dataset. */
// [TARGET listTables(DatasetId, TableListOption...)]
// [VARIABLE "my_project_id"]
// [VARIABLE "my_dataset_name"]
public Page<Table> listTablesFromId(String projectId, String datasetName) {
  // [START bigquery_list_tables]
  DatasetId datasetId = DatasetId.of(projectId, datasetName);
  Page<Table> tables = bigquery.listTables(datasetId, TableListOption.pageSize(100));
  for (Table table : tables.iterateAll()) {
    // do something with the table
  }
  // [END bigquery_list_tables]
  return tables;
}
 

/** Example of listing datasets, specifying the page size. */
// [TARGET listDatasets(DatasetListOption...)]
public Page<Dataset> listDatasets() {
  // [START bigquery_list_datasets]
  // List datasets in the default project
  Page<Dataset> datasets = bigquery.listDatasets(DatasetListOption.pageSize(100));
  for (Dataset dataset : datasets.iterateAll()) {
    // do something with the dataset
  }
  // [END bigquery_list_datasets]
  return datasets;
}
 

/** Example of asynchronously listing monitored resource descriptors, specifying the page size. */
// [TARGET listMonitoredResourceDescriptorsAsync(ListOption...)]
public Page<MonitoredResourceDescriptor> listMonitoredResourceDescriptorsAsync()
    throws ExecutionException, InterruptedException {
  // [START listMonitoredResourceDescriptorsAsync]
  Future<AsyncPage<MonitoredResourceDescriptor>> future =
      logging.listMonitoredResourceDescriptorsAsync(ListOption.pageSize(100));
  // ...
  AsyncPage<MonitoredResourceDescriptor> descriptors = future.get();
  for (MonitoredResourceDescriptor descriptor : descriptors.iterateAll()) {
    // do something with the descriptor
  }
  // [END listMonitoredResourceDescriptorsAsync]
  return descriptors;
}
 

/** Example of asynchronously listing sinks, specifying the page size. */
// [TARGET listSinksAsync(ListOption...)]
public Page<Sink> listSinksAsync() throws ExecutionException, InterruptedException {
  // [START listSinksAsync]
  Future<AsyncPage<Sink>> future = logging.listSinksAsync(ListOption.pageSize(100));
  // ...
  AsyncPage<Sink> sinks = future.get();
  for (Sink sink : sinks.iterateAll()) {
    // do something with the sink
  }
  // [END listSinksAsync]
  return sinks;
}
 

/** Example of listing monitored resource descriptors, specifying the page size. */
// [TARGET listMonitoredResourceDescriptors(ListOption...)]
public Page<MonitoredResourceDescriptor> listMonitoredResourceDescriptors() {
  // [START listMonitoredResourceDescriptors]
  Page<MonitoredResourceDescriptor> descriptors =
      logging.listMonitoredResourceDescriptors(ListOption.pageSize(100));
  for (MonitoredResourceDescriptor descriptor : descriptors.iterateAll()) {
    // do something with the descriptor
  }
  // [END listMonitoredResourceDescriptors]
  return descriptors;
}