com.google.appengine.tools.cloudstorage.GcsFileMetadata Java Examples

@Override
public GcsFileMetadata getObjectMetadata(GcsFilename filename, long timeoutMillis)
    throws IOException {
  ensureInitialized();
  Entity entity;
  try {
    entity = datastore.get(null, makeKey(filename));
    return createGcsFileMetadata(entity, filename);
  } catch (EntityNotFoundException ex1) {
    try {
      entity = datastore.get(null, makeBlobstoreKey(filename));
      return createGcsFileMetadataFromBlobstore(entity, filename);
    } catch (EntityNotFoundException ex2) {
      return null;
    }
  }
}
 

@Override
public void copyObject(GcsFilename source, GcsFilename dest, GcsFileOptions fileOptions,
    long timeoutMillis) throws IOException {
  ensureInitialized();
  GcsFileMetadata meta = getObjectMetadata(source, timeoutMillis);
  if (meta == null) {
    throw new FileNotFoundException(this + ": No such file: " + source);
  }
  if (fileOptions == null) {
    fileOptions = meta.getOptions();
  }

  Token token = beginObjectCreation(dest, fileOptions, timeoutMillis);
  appendFileContentsToToken(source, token);
  finishObjectCreation(token, ByteBuffer.allocate(0), timeoutMillis);
}
 

@Override
public GcsFileMetadata getObjectMetadata(GcsFilename filename, long timeoutMillis)
    throws IOException {
  HTTPRequest req = makeRequest(filename, null, HEAD, timeoutMillis);
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  int responseCode = resp.getResponseCode();
  if (responseCode == 404) {
    return null;
  }
  if (responseCode != 200) {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
  return getMetadataFromResponse(
      filename, resp, getLengthFromHeader(resp, X_GOOG_CONTENT_LENGTH));
}
 

/** Returns {@code true} if a file exists and is non-empty on Google Cloud Storage. */
public boolean existsAndNotEmpty(GcsFilename file) {
  GcsFileMetadata metadata;
  try {
    metadata = gcsService.getMetadata(file);
  } catch (IOException e) {
    logger.atWarning().withCause(e).log("Failed to check if GCS file exists");
    return false;
  }
  return metadata != null && metadata.getLength() > 0;
}
 

/**
 * Traverses the sequence of diff files backwards from checkpointTime and inserts the file
 * metadata into "sequence".  Returns true if a complete sequence was discovered, false if one or
 * more files are missing.
 */
private boolean constructDiffSequence(
    Map<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata,
    DateTime fromTime,
    DateTime lastTime,
    TreeMap<DateTime, GcsFileMetadata> sequence) {
  DateTime checkpointTime = lastTime;
  while (isBeforeOrAt(fromTime, checkpointTime)) {
    GcsFileMetadata metadata;
    if (upperBoundTimesToMetadata.containsKey(checkpointTime)) {
      metadata = Futures.getUnchecked(upperBoundTimesToMetadata.get(checkpointTime));
    } else {
      String filename = DIFF_FILE_PREFIX + checkpointTime;
      logger.atInfo().log("Patching GCS list; discovered file: %s", filename);
      metadata = getMetadata(filename);

      // If we hit a gap, quit.
      if (metadata == null) {
        logger.atInfo().log(
            "Gap discovered in sequence terminating at %s, missing file: %s",
            sequence.lastKey(), filename);
        logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
        return false;
      }
    }
    sequence.put(checkpointTime, metadata);
    checkpointTime = getLowerBoundTime(metadata);
  }
  logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
  return true;
}
 

private GcsFileMetadata getMetadata(String filename) {
  try {
    return gcsService.getMetadata(new GcsFilename(gcsBucket, filename));
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

private Iterable<DateTime> extractTimesFromDiffFiles(List<GcsFileMetadata> diffFiles) {
  return transform(
      diffFiles,
      metadata ->
          DateTime.parse(
              metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length())));
}
 

private GcsFileMetadata createGcsFileMetadataFromBlobstore(Entity entity, GcsFilename filename) {
  return new GcsFileMetadata(
      filename,
      GcsFileOptions.getDefaultInstance(),
      "",
      (Long) entity.getProperty("size"),
      (Date) entity.getProperty("creation"));
}
 

@Test
@InSequence(4)
public void testOptionsAndMetadata() throws IOException {
    GcsFilename filename = new GcsFilename(bucket, OBJECT_NAME + "4");
    GcsFileOptions option = new GcsFileOptions.Builder()
        .acl("public-read")
        .cacheControl("Cache-Control: public, max-age=3600")
        .contentEncoding("Content-Encoding: gzip")
        .contentDisposition("Content-Disposition: attachment")
        .mimeType("text/html")
        .addUserMetadata("userKey", "UserMetadata")
        .build();

    GcsOutputChannel writeChannel = gcsService.createOrReplace(filename, option);
    try (PrintWriter out = new PrintWriter(Channels.newWriter(writeChannel, "UTF8"))) {
        out.println(CONTENT);
        out.flush();
    }

    GcsFileMetadata metaData = gcsService.getMetadata(filename);
    GcsFileOptions option2 = metaData.getOptions();
    try {
        assertEquals(filename, metaData.getFilename());
    } finally {
        gcsService.delete(filename);
    }

    assertEquals("Cache-Control: public, max-age=3600", option2.getCacheControl());
    assertEquals("Content-Encoding: gzip", option2.getContentEncoding());
    assertEquals("Content-Disposition: attachment", option2.getContentDisposition());
    assertEquals("text/html", option2.getMimeType());
    assertEquals("Content-Encoding: gzip", option2.getContentEncoding());
    Map<String, String> userMetadata = option2.getUserMetadata();
    assertEquals(1, userMetadata.size());
    String key = userMetadata.keySet().iterator().next();
    assertEquals("UserMetadata", userMetadata.get(key));
    assertEquals("public-read", option2.getAcl());
}
 

ImmutableList<GcsFileMetadata> listDiffFiles(DateTime fromTime, @Nullable DateTime toTime) {
  logger.atInfo().log("Requested restore from time: %s", fromTime);
  if (toTime != null) {
    logger.atInfo().log("  Until time: %s", toTime);
  }
  // List all of the diff files on GCS and build a map from each file's upper checkpoint time
  // (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with
  // an upper checkpoint time > fromTime.
  TreeMap<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata
      = new TreeMap<>();
  Iterator<ListItem> listItems;
  try {
    // TODO(b/23554360): Use a smarter prefixing strategy to speed this up.
    listItems = gcsService.list(
        gcsBucket,
        new ListOptions.Builder().setPrefix(DIFF_FILE_PREFIX).build());
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  DateTime lastUpperBoundTime = START_OF_TIME;
  while (listItems.hasNext()) {
    final String filename = listItems.next().getName();
    DateTime upperBoundTime = DateTime.parse(filename.substring(DIFF_FILE_PREFIX.length()));
    if (isInRange(upperBoundTime, fromTime, toTime)) {
      upperBoundTimesToMetadata.put(upperBoundTime, executor.submit(() -> getMetadata(filename)));
      lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
    }
  }
  if (upperBoundTimesToMetadata.isEmpty()) {
    logger.atInfo().log("No files found");
    return ImmutableList.of();
  }

  // Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e. the
  // last file that we found) and finding its previous file until we either run out of files or
  // get to one that preceeds "fromTime".
  //
  // GCS file listing is eventually consistent, so it's possible that we are missing a file. The
  // metadata of a file is sufficient to identify the preceding file, so if we start from the
  // last file and work backwards we can verify that we have no holes in our chain (although we
  // may be missing files at the end).
  TreeMap<DateTime, GcsFileMetadata> sequence = new TreeMap<>();
  logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
  boolean inconsistentFileSet = !constructDiffSequence(
      upperBoundTimesToMetadata, fromTime, lastUpperBoundTime, sequence);

  // Verify that all of the elements in the original set are represented in the sequence.  If we
  // find anything that's not represented, construct a sequence for it.
  boolean checkForMoreExtraDiffs = true;  // Always loop at least once.
  while (checkForMoreExtraDiffs) {
    checkForMoreExtraDiffs = false;
    for (DateTime key : upperBoundTimesToMetadata.descendingKeySet()) {
      if (!isInRange(key, fromTime, toTime)) {
        break;
      }
      if (!sequence.containsKey(key)) {
        constructDiffSequence(upperBoundTimesToMetadata, fromTime, key, sequence);
        checkForMoreExtraDiffs = true;
        inconsistentFileSet = true;
        break;
      }
    }
  }

  checkState(
      !inconsistentFileSet,
      "Unable to compute commit diff history, there are either gaps or forks in the history "
      + "file set.  Check log for details.");

  logger.atInfo().log(
      "Actual restore from time: %s", getLowerBoundTime(sequence.firstEntry().getValue()));
  logger.atInfo().log("Found %d files to restore", sequence.size());
  return ImmutableList.copyOf(sequence.values());
}
 

private DateTime getLowerBoundTime(GcsFileMetadata metadata) {
  return DateTime.parse(metadata.getOptions().getUserMetadata().get(LOWER_BOUND_CHECKPOINT));
}
 

@Override
public void run() {
  checkArgument(
      RegistryEnvironment.get() == RegistryEnvironment.ALPHA
          || RegistryEnvironment.get() == RegistryEnvironment.CRASH
          || RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
      "DO NOT RUN ANYWHERE ELSE EXCEPT ALPHA, CRASH OR TESTS.");
  if (dryRun) {
    logger.atInfo().log("Running in dryRun mode");
  }
  List<GcsFileMetadata> diffFiles = diffLister.listDiffFiles(fromTime, toTime);
  if (diffFiles.isEmpty()) {
    logger.atInfo().log("Nothing to restore");
    return;
  }
  Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
  CommitLogCheckpoint lastCheckpoint = null;
  for (GcsFileMetadata metadata : diffFiles) {
    logger.atInfo().log("Restoring: %s", metadata.getFilename().getObjectName());
    try (InputStream input = Channels.newInputStream(
        gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
      PeekingIterator<ImmutableObject> commitLogs =
          peekingIterator(createDeserializingIterator(input));
      lastCheckpoint = (CommitLogCheckpoint) commitLogs.next();
      saveOfy(ImmutableList.of(lastCheckpoint));  // Save the checkpoint itself.
      while (commitLogs.hasNext()) {
        CommitLogManifest manifest = restoreOneTransaction(commitLogs);
        bucketTimestamps.put(manifest.getBucketId(), manifest.getCommitTime());
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
  // Restore the CommitLogCheckpointRoot and CommitLogBuckets.
  saveOfy(
      Streams.concat(
              bucketTimestamps
                  .entrySet()
                  .stream()
                  .map(
                      entry ->
                          new CommitLogBucket.Builder()
                              .setBucketNum(entry.getKey())
                              .setLastWrittenTime(entry.getValue())
                              .build()),
              Stream.of(CommitLogCheckpointRoot.create(lastCheckpoint.getCheckpointTime())))
          .collect(toImmutableList()));
  logger.atInfo().log("Restore complete");
}
 

@Override
public Future<GcsFileMetadata> readObjectAsync(
    ByteBuffer dst, GcsFilename filename, long offset, long timeoutMillis) {
  Preconditions.checkArgument(offset >= 0, "%s: offset must be non-negative: %s", this, offset);
  try {
    ensureInitialized();
    GcsFileMetadata meta = getObjectMetadata(filename, timeoutMillis);
    if (meta == null) {
      return Futures.immediateFailedFuture(
          new FileNotFoundException(this + ": No such file: " + filename));
    }
    if (offset >= meta.getLength()) {
      return Futures.immediateFailedFuture(new BadRangeException(
          "The requested range cannot be satisfied. bytes=" + Long.toString(offset) + "-"
          + Long.toString(offset + dst.remaining()) + " the file is only " + meta.getLength()));
    }

    int read = 0;

    try (ReadableByteChannel readChannel = Channels.newChannel(
        blobStorage.fetchBlob(getBlobKeyForFilename(filename)))) {
      if (offset > 0) {
        long bytesRemaining = offset;
        ByteBuffer seekBuffer = ByteBuffer.allocate(1024);
        while (bytesRemaining > 0) {
          if (bytesRemaining < seekBuffer.limit()) {
            seekBuffer.limit((int) bytesRemaining);
          }
          read = readChannel.read(seekBuffer);
          if (read == -1) {
            return Futures.immediateFailedFuture(new BadRangeException(
                "The requested range cannot be satisfied; seek failed with "
                + Long.toString(bytesRemaining) + " bytes remaining."));

          }
          bytesRemaining -= read;
          seekBuffer.clear();
        }
        read = 0;
      }
      while (read != -1 && dst.hasRemaining()) {
        read = readChannel.read(dst);
      }
    }

    return Futures.immediateFuture(meta);
  } catch (IOException e) {
    return Futures.immediateFailedFuture(e);
  }
}
 

@Override
public ListItemBatch list(String bucket, String prefix, String delimiter,
    String marker, int maxResults, long timeoutMillis) throws IOException {
  ensureInitialized();
  Query query = makeQuery(bucket);
  int prefixLength;
  if (!Strings.isNullOrEmpty(prefix)) {
    Key keyPrefix = makeKey(bucket, prefix);
    query.setFilter(new FilterPredicate(KEY_RESERVED_PROPERTY, GREATER_THAN_OR_EQUAL, keyPrefix));
    prefixLength = prefix.length();
  } else {
    prefixLength = 0;
  }
  FetchOptions fetchOptions = FetchOptions.Builder.withDefaults();
  if (marker != null) {
    fetchOptions.startCursor(Cursor.fromWebSafeString(marker));
  }
  List<ListItem> items = new ArrayList<>(maxResults);
  Set<String> prefixes = new HashSet<>();
  QueryResultIterator<Entity> dsResults =
      datastore.prepare(query).asQueryResultIterator(fetchOptions);
  while (items.size() < maxResults && dsResults.hasNext()) {
    Entity entity = dsResults.next();
    String name = entity.getKey().getName();
    if (prefixLength > 0 && !name.startsWith(prefix)) {
      break;
    }
    if (!Strings.isNullOrEmpty(delimiter)) {
      int delimiterIdx = name.indexOf(delimiter, prefixLength);
      if (delimiterIdx > 0) {
        name = name.substring(0, delimiterIdx + 1);
        if (prefixes.add(name)) {
          items.add(new ListItem.Builder().setName(name).setDirectory(true).build());
        }
        continue;
      }
    }
    GcsFilename filename = new GcsFilename(bucket, name);
    GcsFileMetadata metadata = createGcsFileMetadata(entity, filename);
    ListItem listItem = new ListItem.Builder()
        .setName(name)
        .setLength(metadata.getLength())
        .setLastModified(metadata.getLastModified())
        .build();
    items.add(listItem);
  }
  Cursor cursor = dsResults.getCursor();
  String nextMarker = null;
  if (items.size() == maxResults && cursor != null) {
    nextMarker = cursor.toWebSafeString();
  }
  return new ListItemBatch(items, nextMarker);
}
 

/**
 * Might not fill all of dst.
 */
@Override
public Future<GcsFileMetadata> readObjectAsync(final ByteBuffer dst, final GcsFilename filename,
    long startOffsetBytes, long timeoutMillis) {
  Preconditions.checkArgument(startOffsetBytes >= 0, "%s: offset must be non-negative: %s", this,
      startOffsetBytes);
  final int n = dst.remaining();
  Preconditions.checkArgument(n > 0, "%s: dst full: %s", this, dst);
  final int want = Math.min(READ_LIMIT_BYTES, n);

  final HTTPRequest req = makeRequest(filename, null, GET, timeoutMillis);
  req.setHeader(
      new HTTPHeader(RANGE, "bytes=" + startOffsetBytes + "-" + (startOffsetBytes + want - 1)));
  final HTTPRequestInfo info = new HTTPRequestInfo(req);
  return new FutureWrapper<HTTPResponse, GcsFileMetadata>(urlfetch.fetchAsync(req)) {
    @Override
    protected GcsFileMetadata wrap(HTTPResponse resp) throws IOException {
      long totalLength;
      switch (resp.getResponseCode()) {
        case 200:
          totalLength = getLengthFromHeader(resp, X_GOOG_CONTENT_LENGTH);
          break;
        case 206:
          totalLength = getLengthFromContentRange(resp);
          break;
        case 404:
          throw new FileNotFoundException("Could not find: " + filename);
        case 416:
          throw new BadRangeException("Requested Range not satisfiable; perhaps read past EOF? "
              + URLFetchUtils.describeRequestAndResponse(info, resp));
        default:
          throw HttpErrorHandler.error(info, resp);
      }
      byte[] content = resp.getContent();
      Preconditions.checkState(content.length <= want, "%s: got %s > wanted %s", this,
          content.length, want);
      dst.put(content);
      return getMetadataFromResponse(filename, resp, totalLength);
    }

    @Override
    protected Throwable convertException(Throwable e) {
      return OauthRawGcsService.convertException(info, e);
    }
  };
}