com.google.api.services.storage.model.Objects Java Examples

@Test
public void testNonExistentObjectReturnsEmptyResult() throws IOException {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
  Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);

  GcsPath pattern = GcsPath.fromUri("gs://testbucket/testdirectory/nonexistentfile");
  GoogleJsonResponseException expectedException =
      googleJsonResponseException(
          HttpStatusCodes.STATUS_CODE_NOT_FOUND, "It don't exist", "Nothing here to see");

  when(mockStorage.objects()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get(pattern.getBucket(), pattern.getObject()))
      .thenReturn(mockStorageGet);
  when(mockStorageGet.execute()).thenThrow(expectedException);

  assertEquals(Collections.emptyList(), gcsUtil.expand(pattern));
}
 

public void updateMetadata(GoogleCloudStorageItemInfo itemInfo, Map<String, byte[]> metadata)
    throws IOException {
  StorageResourceId resourceId = itemInfo.getResourceId();
  Preconditions.checkArgument(
      resourceId.isStorageObject(), "Expected full StorageObject ID, got %s", resourceId);

  StorageObject storageObject = new StorageObject().setMetadata(encodeMetadata(metadata));

  Storage.Objects.Patch patchObject =
      configureRequest(
              gcs.objects()
                  .patch(resourceId.getBucketName(), resourceId.getObjectName(), storageObject),
              resourceId.getBucketName())
          .setIfMetagenerationMatch(itemInfo.getMetaGeneration());

  patchObject.execute();
}
 

@Override
public void createEmptyObject(StorageResourceId resourceId, CreateObjectOptions options)
    throws IOException {
  Preconditions.checkArgument(
      resourceId.isStorageObject(), "Expected full StorageObject id, got %s", resourceId);

  Storage.Objects.Insert insertObject = prepareEmptyInsert(resourceId, options);
  try {
    insertObject.execute();
  } catch (IOException ioe) {
    if (canIgnoreExceptionForEmptyObject(ioe, resourceId, options)) {
      logger.atInfo().withCause(ioe).log(
          "Ignoring exception; verified object already exists with desired state.");
    } else {
      throw ioe;
    }
  }
}
 

/**
 * Uploads data to an object in a bucket.
 *
 * @param name the name of the destination object.
 * @param contentType the MIME type of the data.
 * @param file the file to upload.
 * @param bucketName the name of the bucket to create the object in.
 */
public static void uploadFile(String name, String contentType, File file, String bucketName)
    throws IOException, GeneralSecurityException {
  InputStreamContent contentStream =
      new InputStreamContent(contentType, new FileInputStream(file));
  // Setting the length improves upload performance
  contentStream.setLength(file.length());
  StorageObject objectMetadata =
      new StorageObject()
          // Set the destination object name
          .setName(name)
          // Set the access control list to publicly read-only
          .setAcl(
              Arrays.asList(new ObjectAccessControl().setEntity("allUsers").setRole("READER")));

  // Do the insert
  Storage client = StorageFactory.getService();
  Storage.Objects.Insert insertRequest =
      client.objects().insert(bucketName, objectMetadata, contentStream);

  insertRequest.execute();
}
 

@Test
public void testMakeRewriteOps() throws IOException {
  GcsOptions gcsOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = gcsOptions.getGcsUtil();

  LinkedList<RewriteOp> rewrites =
      gcsUtil.makeRewriteOps(makeStrings("s", 1), makeStrings("d", 1));
  assertEquals(1, rewrites.size());

  RewriteOp rewrite = rewrites.pop();
  assertTrue(rewrite.getReadyToEnqueue());
  Storage.Objects.Rewrite request = rewrite.rewriteRequest;
  assertNull(request.getMaxBytesRewrittenPerCall());
  assertEquals("bucket", request.getSourceBucket());
  assertEquals("s0", request.getSourceObject());
  assertEquals("bucket", request.getDestinationBucket());
  assertEquals("d0", request.getDestinationObject());
}
 

@Test
public void testFileSizeNonBatch() throws Exception {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
  Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);

  when(mockStorage.objects()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get("testbucket", "testobject")).thenReturn(mockStorageGet);
  when(mockStorageGet.execute())
      .thenReturn(new StorageObject().setSize(BigInteger.valueOf(1000)));

  assertEquals(1000, gcsUtil.fileSize(GcsPath.fromComponents("testbucket", "testobject")));
}
 

/**
 * Test successful operation of GoogleCloudStorage.copy(4) where srcBucketName == dstBucketName.
 */
@Test
public void testCopyObjectsNormalOperationSameBucket() throws IOException {
  String dstObject = OBJECT_NAME + "-copy";
  StorageObject object = newStorageObject(BUCKET_NAME, dstObject);
  MockHttpTransport transport =
      mockTransport(jsonDataResponse(new Objects().setItems(ImmutableList.of(object))));

  GoogleCloudStorage gcs = mockedGcs(transport);

  gcs.copy(BUCKET_NAME, ImmutableList.of(OBJECT_NAME), BUCKET_NAME, ImmutableList.of(dstObject));

  assertThat(trackingHttpRequestInitializer.getAllRequestStrings())
      .containsExactly(
          copyRequestString(BUCKET_NAME, OBJECT_NAME, BUCKET_NAME, dstObject, "copyTo"))
      .inOrder();
}
 

/** Lists {@link Objects} given the {@code bucket}, {@code prefix}, {@code pageToken}. */
public Objects listObjects(String bucket, String prefix, @Nullable String pageToken)
    throws IOException {
  // List all objects that start with the prefix (including objects in sub-directories).
  Storage.Objects.List listObject = storageClient.objects().list(bucket);
  listObject.setMaxResults(MAX_LIST_ITEMS_PER_CALL);
  listObject.setPrefix(prefix);

  if (pageToken != null) {
    listObject.setPageToken(pageToken);
  }

  try {
    return ResilientOperation.retry(
        ResilientOperation.getGoogleRequestCallable(listObject),
        createBackOff(),
        RetryDeterminer.SOCKET_ERRORS,
        IOException.class);
  } catch (Exception e) {
    throw new IOException(
        String.format("Unable to match files in bucket %s, prefix %s.", bucket, prefix), e);
  }
}
 

private void setUpGetFilesPage(List<String> objectNames) {
  // these are final classes, so use fakes instead of mocks.
  List<StorageObject> fakeItems = new ArrayList<>();
  for (int i = 0; i < objectNames.size(); ++i) {
    StorageObject fakeObject = new StorageObject().setName(objectNames.get(i));
    fakeItems.add(fakeObject);
  }

  Objects listObjects = new Objects().setItems(fakeItems);

  try {
    when(this.objectList.execute()).thenReturn(listObjects);
  } catch (IOException e) {
    fail("Failed to setup getFilesPage");
  }
}
 

/** Test successful operation of GoogleCloudStorage.listObjectNames(3). */
@Test
public void testListObjectNamesPrefix() throws IOException {
  String prefix = "foo/bar/baz/";
  String pageToken = "pageToken_0";

  MockHttpTransport transport =
      mockTransport(
          jsonDataResponse(
              new Objects()
                  .setPrefixes(ImmutableList.of("foo/bar/baz/dir0/", "foo/bar/baz/dir1/"))
                  .setNextPageToken(pageToken)),
          jsonDataResponse(
              new Objects()
                  .setItems(
                      ImmutableList.of(
                          newStorageObject(BUCKET_NAME, "foo/bar/baz/"),
                          newStorageObject(BUCKET_NAME, "foo/bar/baz/obj0"),
                          newStorageObject(BUCKET_NAME, "foo/bar/baz/obj1")))
                  .setNextPageToken(null)));

  GoogleCloudStorage gcs = mockedGcs(transport);

  List<String> objectNames = gcs.listObjectNames(BUCKET_NAME, prefix, PATH_DELIMITER);

  assertThat(objectNames)
      .containsExactly(
          "foo/bar/baz/dir0/", "foo/bar/baz/dir1/", "foo/bar/baz/obj0", "foo/bar/baz/obj1")
      .inOrder();
  assertThat(trackingHttpRequestInitializer.getAllRequestStrings())
      .containsExactly(
          listRequestString(BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null),
          listRequestString(BUCKET_NAME, prefix, /* maxResults= */ 1024, pageToken))
      .inOrder();
}
 

<RequestT extends StorageRequest<?>> RequestT configureRequest(
    RequestT request, String bucketName) {
  setRequesterPaysProject(request, bucketName);

  if (request instanceof Storage.Objects.Get || request instanceof Storage.Objects.Insert) {
    setEncryptionHeaders(request);
  }

  if (request instanceof Storage.Objects.Rewrite || request instanceof Storage.Objects.Copy) {
    setEncryptionHeaders(request);
    setDecryptionHeaders(request);
  }

  return request;
}
 

private void purgeOldVersions(String path) throws Exception {
  Storage.Objects.List listObjects = obj_api.list(bucketName).setPrefix(path).setVersions(true);

  com.google.api.services.storage.model.Objects objects;

  // Keep the 0th object on the first page (which is current).
  List<Long> generations = new ArrayList(32);
  do {
    objects = timeExecute(listTimer, listObjects);
    List<StorageObject> items = objects.getItems();
    if (items != null) {
      int n = items.size();
      while (--n >= 0) {
        generations.add(items.get(n).getGeneration());
      }
    }
    listObjects.setPageToken(objects.getNextPageToken());
  } while (objects.getNextPageToken() != null);

  for (long generation : generations) {
    if (generation == generations.get(0)) {
      continue;
    }
    log.debug("Remove {} generation {}", value("path", path), value("generation", generation));
    timeExecute(purgeTimer, obj_api.delete(bucketName, path).setGeneration(generation));
  }
}
 

@Override
public ListPage<GoogleCloudStorageItemInfo> listObjectInfoPage(
    String bucketName, String objectNamePrefix, String delimiter, String pageToken)
    throws IOException {
  logger.atFine().log(
      "listObjectInfoPage(%s, %s, %s, %s)", bucketName, objectNamePrefix, delimiter, pageToken);

  Storage.Objects.List listObject =
      createListRequest(
          bucketName,
          objectNamePrefix,
          delimiter,
          /* includeTrailingDelimiter= */ true,
          MAX_RESULTS_UNLIMITED);
  if (pageToken != null) {
    logger.atFine().log("listObjectInfoPage: next page %s", pageToken);
    listObject.setPageToken(pageToken);
  }

  // Helper will handle going through pages of list results and accumulating them.
  List<StorageObject> listedObjects = new ArrayList<>();
  List<String> listedPrefixes = new ArrayList<>();
  String nextPageToken =
      listStorageObjectsAndPrefixesPage(
          listObject, MAX_RESULTS_UNLIMITED, listedObjects, listedPrefixes);

  // For the listedObjects, we simply parse each item into a GoogleCloudStorageItemInfo without
  // further work.
  List<GoogleCloudStorageItemInfo> objectInfos = new ArrayList<>(listedObjects.size());
  for (StorageObject obj : listedObjects) {
    objectInfos.add(
        createItemInfoForStorageObject(new StorageResourceId(bucketName, obj.getName()), obj));
  }

  if (!listedPrefixes.isEmpty()) {
    handlePrefixes(bucketName, listedPrefixes, objectInfos);
  }

  return new ListPage<>(objectInfos, nextPageToken);
}
 

/** Test GoogleCloudStorage.listObjectNames(3) with maxResults set. */
@Test
public void testListObjectNamesPrefixLimited() throws IOException {
  String prefix = "foo/bar/baz/";
  int maxResults = 3;

  MockHttpTransport transport =
      mockTransport(
          jsonDataResponse(
              new Objects()
                  .setPrefixes(ImmutableList.of("foo/bar/baz/dir0/", "foo/bar/baz/dir1/"))
                  .setItems(
                      ImmutableList.of(
                          newStorageObject(BUCKET_NAME, "foo/bar/baz/"),
                          newStorageObject(BUCKET_NAME, "foo/bar/baz/obj0")))
                  .setNextPageToken("pageToken0")));

  GoogleCloudStorage gcs = mockedGcs(transport);

  List<String> objectNames = gcs.listObjectNames(BUCKET_NAME, prefix, PATH_DELIMITER, maxResults);

  assertThat(objectNames)
      .containsExactly("foo/bar/baz/dir0/", "foo/bar/baz/dir1/", "foo/bar/baz/obj0")
      .inOrder();
  assertThat(trackingHttpRequestInitializer.getAllRequestStrings())
      .containsExactly(
          listRequestString(BUCKET_NAME, prefix, maxResults + 1, /* pageToken= */ null))
      .inOrder();
}
 

private Storage.Objects.List createListRequest(
    String bucketName,
    String objectNamePrefix,
    String delimiter,
    boolean includeTrailingDelimiter,
    long maxResults)
    throws IOException {
  logger.atFine().log(
      "createListRequest(%s, %s, %s, %s, %d)",
      bucketName, objectNamePrefix, delimiter, includeTrailingDelimiter, maxResults);
  checkArgument(!Strings.isNullOrEmpty(bucketName), "bucketName must not be null or empty");

  Storage.Objects.List listObject = configureRequest(gcs.objects().list(bucketName), bucketName);

  // Set delimiter if supplied.
  if (delimiter != null) {
    listObject.setDelimiter(delimiter);
    listObject.setIncludeTrailingDelimiter(includeTrailingDelimiter);
  }

  // Set number of items to retrieve per call.
  if (maxResults <= 0 || maxResults + 1 >= storageOptions.getMaxListItemsPerCall()) {
    listObject.setMaxResults(storageOptions.getMaxListItemsPerCall());
  } else {
    // We add one in case we filter out objectNamePrefix.
    listObject.setMaxResults(maxResults + 1);
  }

  // Set prefix if supplied.
  if (!Strings.isNullOrEmpty(objectNamePrefix)) {
    listObject.setPrefix(objectNamePrefix);
  }

  return listObject;
}
 

@Test
public void testListObjectInfoBasic() throws IOException {
  String prefix = "foo/bar/baz/";

  StorageObject object1 = newStorageObject(BUCKET_NAME, "foo/bar/baz/obj0");
  StorageObject object2 = newStorageObject(BUCKET_NAME, "foo/bar/baz/obj1");

  List<StorageObject> objects =
      ImmutableList.of(newStorageObject(BUCKET_NAME, "foo/bar/baz/"), object1, object2);

  MockHttpTransport transport =
      mockTransport(jsonDataResponse(new Objects().setItems(objects).setNextPageToken(null)));

  GoogleCloudStorage gcs = mockedGcs(transport);

  List<GoogleCloudStorageItemInfo> objectInfos =
      gcs.listObjectInfo(BUCKET_NAME, prefix, PATH_DELIMITER);

  // The item exactly matching the input prefix will be discarded.
  assertThat(objectInfos)
      .containsExactly(
          createItemInfoForStorageObject(
              new StorageResourceId(BUCKET_NAME, object1.getName()), object1),
          createItemInfoForStorageObject(
              new StorageResourceId(BUCKET_NAME, object2.getName()), object2))
      .inOrder();
  assertThat(trackingHttpRequestInitializer.getAllRequestStrings())
      .containsExactly(
          listRequestString(
              BUCKET_NAME,
              /* includeTrailingDelimiter= */ true,
              prefix,
              /* maxResults= */ 1024,
              /* pageToken= */ null))
      .inOrder();
}
 

/**
 * Helper for both listObjectNames and listObjectInfo that executes the actual API calls to get
 * paginated lists, accumulating the StorageObjects and String prefixes into the params {@code
 * listedObjects} and {@code listedPrefixes}.
 *
 * @param bucketName bucket name
 * @param objectNamePrefix object name prefix or null if all objects in the bucket are desired
 * @param delimiter delimiter to use (typically "/"), otherwise null
 * @param includeTrailingDelimiter whether to include prefix objects into the {@code
 *     listedObjects}
 * @param maxResults maximum number of results to return (total of both {@code listedObjects} and
 *     {@code listedPrefixes}), unlimited if negative or zero
 * @param listedObjects output parameter into which retrieved StorageObjects will be added
 * @param listedPrefixes output parameter into which retrieved prefixes will be added
 */
private void listStorageObjectsAndPrefixes(
    String bucketName,
    String objectNamePrefix,
    String delimiter,
    boolean includeTrailingDelimiter,
    long maxResults,
    List<StorageObject> listedObjects,
    List<String> listedPrefixes)
    throws IOException {
  logger.atFine().log(
      "listStorageObjectsAndPrefixes(%s, %s, %s, %s, %d)",
      bucketName, objectNamePrefix, delimiter, includeTrailingDelimiter, maxResults);

  checkArgument(
      listedObjects != null && listedObjects.isEmpty(),
      "Must provide a non-null empty container for listedObjects.");
  checkArgument(
      listedPrefixes != null && listedPrefixes.isEmpty(),
      "Must provide a non-null empty container for listedPrefixes.");

  Storage.Objects.List listObject =
      createListRequest(
          bucketName, objectNamePrefix, delimiter, includeTrailingDelimiter, maxResults);

  String pageToken = null;
  do {
    if (pageToken != null) {
      logger.atFine().log("listStorageObjectsAndPrefixes: next page %s", pageToken);
      listObject.setPageToken(pageToken);
    }
    pageToken =
        listStorageObjectsAndPrefixesPage(listObject, maxResults, listedObjects, listedPrefixes);
  } while (pageToken != null
      && getMaxRemainingResults(maxResults, listedPrefixes, listedObjects) > 0);
}
 

/**
 * Helper for creating a Storage.Objects.Copy object ready for dispatch given a bucket and object
 * for an empty object to be created. Caller must already verify that {@code resourceId}
 * represents a StorageObject and not a bucket.
 */
private Storage.Objects.Insert prepareEmptyInsert(
    StorageResourceId resourceId, CreateObjectOptions createObjectOptions) throws IOException {
  Map<String, String> rewrittenMetadata = encodeMetadata(createObjectOptions.getMetadata());
  StorageObject object =
      new StorageObject()
          .setName(resourceId.getObjectName())
          .setMetadata(rewrittenMetadata)
          .setContentEncoding(createObjectOptions.getContentEncoding());

  // Ideally we'd use EmptyContent, but Storage requires an AbstractInputStreamContent and not
  // just an HttpContent, so we'll just use the next easiest thing.
  ByteArrayContent emptyContent =
      new ByteArrayContent(createObjectOptions.getContentType(), new byte[0]);
  Storage.Objects.Insert insertObject =
      configureRequest(
          gcs.objects().insert(resourceId.getBucketName(), object, emptyContent),
          resourceId.getBucketName());
  insertObject.setDisableGZipContent(true);
  clientRequestHelper.setDirectUploadEnabled(insertObject, true);

  if (resourceId.hasGenerationId()) {
    insertObject.setIfGenerationMatch(resourceId.getGenerationId());
  } else if (!createObjectOptions.overwriteExisting()) {
    insertObject.setIfGenerationMatch(0L);
  }
  return insertObject;
}
 

@Test
public void testListObjectInfoReturnPrefixes() throws IOException {
  String objectPrefix = "foo/bar/baz/";
  String delimiter = "/";
  String dir0Name = "foo/bar/baz/dir0/";
  String dir1Name = "foo/bar/baz/dir1/";
  StorageObject dir0 = newStorageObject(BUCKET_NAME, dir0Name);
  StorageObject dir1 = newStorageObject(BUCKET_NAME, dir1Name);

  List<StorageObject> objects =
      ImmutableList.of(newStorageObject(BUCKET_NAME, objectPrefix), dir0, dir1);

  MockHttpTransport transport =
      mockTransport(jsonDataResponse(new Objects().setItems(objects).setNextPageToken(null)));

  GoogleCloudStorage gcs = mockedGcs(transport);

  List<GoogleCloudStorageItemInfo> objectInfos =
      gcs.listObjectInfo(BUCKET_NAME, objectPrefix, delimiter);

  trackingHttpRequestInitializer.getAllRequestStrings();
  assertThat(objectInfos)
      .containsExactly(
          createItemInfoForStorageObject(new StorageResourceId(BUCKET_NAME, dir0Name), dir0),
          createItemInfoForStorageObject(new StorageResourceId(BUCKET_NAME, dir1Name), dir1))
      .inOrder();

  assertThat(trackingHttpRequestInitializer.getAllRequestStrings())
      .containsExactly(
          listRequestString(
              BUCKET_NAME,
              /* includeTrailingDelimiter= */ true,
              objectPrefix,
              /* maxResults= */ 1024,
              /* pageToken= */ null))
      .inOrder();
}
 

private <T> T deserialize(Storage storage, StorageObject object, TypeReference<T> typeReference)
    throws IOException {
  ByteArrayOutputStream output = new java.io.ByteArrayOutputStream();
  Storage.Objects.Get getter = storage.objects().get(object.getBucket(), object.getName());
  getter.executeMediaAndDownloadTo(output);
  String json = output.toString("UTF8");

  return kayentaObjectMapper.readValue(json, typeReference);
}
 

@Test
public void testMakeRewriteOpsWithOptions() throws IOException {
  GcsOptions gcsOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = gcsOptions.getGcsUtil();
  gcsUtil.maxBytesRewrittenPerCall = 1337L;

  LinkedList<RewriteOp> rewrites =
      gcsUtil.makeRewriteOps(makeStrings("s", 1), makeStrings("d", 1));
  assertEquals(1, rewrites.size());

  RewriteOp rewrite = rewrites.pop();
  assertTrue(rewrite.getReadyToEnqueue());
  Storage.Objects.Rewrite request = rewrite.rewriteRequest;
  assertEquals(Long.valueOf(1337L), request.getMaxBytesRewrittenPerCall());
}
 

@Test
public void testRetryFileSizeNonBatch() throws IOException {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
  Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);

  BackOff mockBackOff =
      BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.withMaxRetries(2).backoff());

  when(mockStorage.objects()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get("testbucket", "testobject")).thenReturn(mockStorageGet);
  when(mockStorageGet.execute())
      .thenThrow(new SocketTimeoutException("SocketException"))
      .thenThrow(new SocketTimeoutException("SocketException"))
      .thenReturn(new StorageObject().setSize(BigInteger.valueOf(1000)));

  assertEquals(
      1000,
      gcsUtil
          .getObject(
              GcsPath.fromComponents("testbucket", "testobject"),
              mockBackOff,
              new FastNanoClockAndSleeper())
          .getSize()
          .longValue());
  assertEquals(BackOff.STOP, mockBackOff.nextBackOffMillis());
}
 

@Test
public void testAccessDeniedObjectThrowsIOException() throws IOException {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
  Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);

  GcsPath pattern = GcsPath.fromUri("gs://testbucket/testdirectory/accessdeniedfile");
  GoogleJsonResponseException expectedException =
      googleJsonResponseException(
          HttpStatusCodes.STATUS_CODE_FORBIDDEN,
          "Waves hand mysteriously",
          "These aren't the buckets you're looking for");

  when(mockStorage.objects()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get(pattern.getBucket(), pattern.getObject()))
      .thenReturn(mockStorageGet);
  when(mockStorageGet.execute()).thenThrow(expectedException);

  thrown.expect(IOException.class);
  thrown.expectMessage("Unable to get the file object for path");
  gcsUtil.expand(pattern);
}
 

/**
 * Expands a pattern into {@link MatchResult}.
 *
 * @throws IllegalArgumentException if {@code gcsPattern} does not contain globs.
 */
@VisibleForTesting
MatchResult expand(GcsPath gcsPattern) throws IOException {
  String prefix = GcsUtil.getNonWildcardPrefix(gcsPattern.getObject());
  Pattern p = Pattern.compile(GcsUtil.wildcardToRegexp(gcsPattern.getObject()));

  LOG.debug(
      "matching files in bucket {}, prefix {} against pattern {}",
      gcsPattern.getBucket(),
      prefix,
      p.toString());

  String pageToken = null;
  List<Metadata> results = new ArrayList<>();
  do {
    Objects objects = options.getGcsUtil().listObjects(gcsPattern.getBucket(), prefix, pageToken);
    if (objects.getItems() == null) {
      break;
    }

    // Filter objects based on the regex.
    for (StorageObject o : objects.getItems()) {
      String name = o.getName();
      // Skip directories, which end with a slash.
      if (p.matcher(name).matches() && !name.endsWith("/")) {
        LOG.debug("Matched object: {}", name);
        results.add(toMetadata(o));
      }
    }
    pageToken = objects.getNextPageToken();
  } while (pageToken != null);
  return MatchResult.create(Status.OK, results);
}
 

private void setUpGetFilesPage(List<String> objectNames) {
  // these are final classes, so use fakes instead of mocks.
  List<StorageObject> fakeItems = new ArrayList<>();
  for (String anObjectName : objectNames) {
    fakeItems.add(new StorageObject().setName(anObjectName));
  }

  Objects listObjects = new Objects().setItems(fakeItems);

  try {
    when(this.objectList.execute()).thenReturn(listObjects);
  } catch (IOException e) {
    fail("Failed to setup getFilesPage");
  }
}
 

private void setUpGetFilesPage(List<String> objectNames) {
  // these are final classes, so use fakes instead of mocks.
  List<StorageObject> fakeItems = new ArrayList<>();
  for (String anObjectName : objectNames) {
    fakeItems.add(new StorageObject().setName(anObjectName));
  }

  Objects listObjects = new Objects().setItems(fakeItems);

  try {
    when(this.objectList.execute()).thenReturn(listObjects);
  } catch (IOException e) {
    fail("Failed to setup getFilesPage");
  }
}
 

private void setUpGetFilesPage(List<String> objectNames) {
  // these are final classes, so use fakes instead of mocks.
  List<StorageObject> fakeItems = new ArrayList<>();
  for (String anObjectName : objectNames) {
    fakeItems.add(new StorageObject().setName(anObjectName));
  }

  Objects listObjects = new Objects().setItems(fakeItems);

  try {
    when(this.objectList.execute()).thenReturn(listObjects);
  } catch (IOException e) {
    fail("Failed to setup getFilesPage");
  }
}
 

/**
 * Obtain a list of files that reside in the repository directory.
 * This will yield all files, no matter how deeply nested they are in the directory.
 * @param fileNames The list which will be populated with the file names.
 * @param nextPageToken The token used to get the next page of the file list.
 * @return The token which will be used to get the next page of the file list.
 * @throws GeneralSecurityException Thrown if there's a permissions error using the GCS API.
 * @throws IOException Thrown if there's a IO error using the GCS API.
 */
String getFilesPage(List<String> fileNames, String nextPageToken)
    throws GeneralSecurityException, IOException, BucketAccessException {
  Objects objects =
      getStorageApiStub()
          .objects()
          .list(this.bucket)
          .setPageToken(nextPageToken)
          .setPrefix(this.repository + this.getDirDelimiter())
          .execute();
  if (objects == null || objects.getItems() == null) {
    String messageFormat = new StringBuilder()
        .append("Can't access objects in gs://%s/%s\n\n")
        .append("Check that your appengine-web.xml variables are setup correctly.\n")
        .append("In particular, check the correctness of the following variables:\n\n")
        .append("    POLICY_SCANNER_ORG_NAME\n")
        .append("    POLICY_SCANNER_INPUT_REPOSITORY_URL\n").toString();
    throw new BucketAccessException(String.format(messageFormat, this.bucket, this.repository));
  }
  for (int i = 0; i < objects.getItems().size(); ++i) {
    String fileName = objects.getItems().get(i).getName();
    if (!fileName.endsWith(getDirDelimiter())) {
      fileNames.add(fileName);
    }
  }
  return objects.getNextPageToken();
}
 

/** See {@link GoogleCloudStorage#open(StorageResourceId)} for details about expected behavior. */
@Override
public SeekableByteChannel open(
    final StorageResourceId resourceId, GoogleCloudStorageReadOptions readOptions)
    throws IOException {
  logger.atFine().log("open(%s, %s)", resourceId, readOptions);
  Preconditions.checkArgument(
      resourceId.isStorageObject(), "Expected full StorageObject id, got %s", resourceId);

  if (storageOptions.isGrpcEnabled()) {
    return GoogleCloudStorageGrpcReadChannel.open(
        storageStubProvider.getBlockingStub(),
        resourceId.getBucketName(),
        resourceId.getObjectName(),
        readOptions);
  }

  // The underlying channel doesn't initially read data, which means that we won't see a
  // FileNotFoundException until read is called. As a result, in order to find out if the object
  // exists, we'll need to do an RPC (metadata or data). A metadata check should be a less
  // expensive operation than a read data operation.
  GoogleCloudStorageItemInfo info;
  if (readOptions.getFastFailOnNotFound()) {
    info = getItemInfo(resourceId);
    if (!info.exists()) {
      throw createFileNotFoundException(
          resourceId.getBucketName(), resourceId.getObjectName(), /* cause= */ null);
    }
  } else {
    info = null;
  }

  return new GoogleCloudStorageReadChannel(
      gcs, resourceId, errorExtractor, clientRequestHelper, readOptions) {

    @Override
    @Nullable
    protected GoogleCloudStorageItemInfo getInitialMetadata() {
      return info;
    }

    @Override
    protected Storage.Objects.Get createRequest() throws IOException {
      return configureRequest(super.createRequest(), resourceId.getBucketName());
    }
  };
}
 

@Test
public void testMatch() throws Exception {
  Objects modelObjects = new Objects();
  List<StorageObject> items = new ArrayList<>();
  // A directory
  items.add(new StorageObject().setBucket("testbucket").setName("testdirectory/"));

  // Files within the directory
  items.add(createStorageObject("gs://testbucket/testdirectory/file1name", 1L /* fileSize */));
  items.add(createStorageObject("gs://testbucket/testdirectory/file2name", 2L /* fileSize */));
  items.add(createStorageObject("gs://testbucket/testdirectory/file3name", 3L /* fileSize */));
  items.add(createStorageObject("gs://testbucket/testdirectory/file4name", 4L /* fileSize */));
  items.add(createStorageObject("gs://testbucket/testdirectory/otherfile", 5L /* fileSize */));
  items.add(createStorageObject("gs://testbucket/testdirectory/anotherfile", 6L /* fileSize */));

  modelObjects.setItems(items);
  when(mockGcsUtil.listObjects(eq("testbucket"), anyString(), isNull(String.class)))
      .thenReturn(modelObjects);

  List<GcsPath> gcsPaths =
      ImmutableList.of(
          GcsPath.fromUri("gs://testbucket/testdirectory/non-exist-file"),
          GcsPath.fromUri("gs://testbucket/testdirectory/otherfile"));

  when(mockGcsUtil.getObjects(eq(gcsPaths)))
      .thenReturn(
          ImmutableList.of(
              StorageObjectOrIOException.create(new FileNotFoundException()),
              StorageObjectOrIOException.create(
                  createStorageObject("gs://testbucket/testdirectory/otherfile", 4L))));

  List<String> specs =
      ImmutableList.of(
          "gs://testbucket/testdirectory/file[1-3]*",
          "gs://testbucket/testdirectory/non-exist-file",
          "gs://testbucket/testdirectory/otherfile");
  List<MatchResult> matchResults = gcsFileSystem.match(specs);
  assertEquals(3, matchResults.size());
  assertEquals(Status.OK, matchResults.get(0).status());
  assertThat(
      ImmutableList.of(
          "gs://testbucket/testdirectory/file1name",
          "gs://testbucket/testdirectory/file2name",
          "gs://testbucket/testdirectory/file3name"),
      contains(toFilenames(matchResults.get(0)).toArray()));
  assertEquals(Status.NOT_FOUND, matchResults.get(1).status());
  assertEquals(Status.OK, matchResults.get(2).status());
  assertThat(
      ImmutableList.of("gs://testbucket/testdirectory/otherfile"),
      contains(toFilenames(matchResults.get(2)).toArray()));
}