com.google.appengine.tools.cloudstorage.GcsFileOptions Java Examples

The following examples show how to use com.google.appengine.tools.cloudstorage.GcsFileOptions. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: GCSClientTest.java    From appengine-tck with Apache License 2.0 6 votes vote down vote up
@Test
@InSequence(1)
public void testCreateGsObj() throws IOException {
    GcsFilename filename = new GcsFilename(bucket, OBJECT_NAME);
    GcsFileOptions option = new GcsFileOptions.Builder()
        .mimeType("text/html")
        .acl("public-read")
        .build();

    try (GcsOutputChannel writeChannel = gcsService.createOrReplace(filename, option)) {
        PrintWriter out = new PrintWriter(Channels.newWriter(writeChannel, "UTF8"));
        out.println(CONTENT);
        out.flush();

        writeChannel.waitForOutstandingWrites();
        writeChannel.write(ByteBuffer.wrap(MORE_WORDS.getBytes()));
        assertEquals(filename, writeChannel.getFilename());
    }

    assertEquals(bucket, filename.getBucketName());
    assertEquals(OBJECT_NAME, filename.getObjectName());
}
 
Example #2
Source File: ObjectifyStorageIo.java    From appinventor-extensions with Apache License 2.0 6 votes vote down vote up
private FileData createRawFile(Key<ProjectData> projectKey, FileData.RoleEnum role,
  String userId, String fileName, byte[] content) throws ObjectifyException, IOException {
  validateGCS();
  FileData file = new FileData();
  file.fileName = fileName;
  file.projectKey = projectKey;
  file.role = role;
  file.userId = userId;
  if (useGCSforFile(fileName, content.length)) {
    file.isGCS = true;
    file.gcsName = makeGCSfileName(fileName, projectKey.getId());
    GcsOutputChannel outputChannel =
      gcsService.createOrReplace(new GcsFilename(GCS_BUCKET_NAME, file.gcsName), GcsFileOptions.getDefaultInstance());
    outputChannel.write(ByteBuffer.wrap(content));
    outputChannel.close();
  } else {
    file.content = content;
  }
  return file;
}
 
Example #3
Source File: GcsBlobstoreUploadTest.java    From appengine-tck with Apache License 2.0 6 votes vote down vote up
@Test
@InSequence(100)
public void testCreateGsBlobKey() throws Exception {
    final long ts = System.currentTimeMillis();
    final byte[] bytes = "FooBar".getBytes();

    GcsService service = GcsServiceFactory.createGcsService();
    GcsFilename filename = new GcsFilename("GcsBucket", String.valueOf(ts));
    GcsFileOptions options = new GcsFileOptions.Builder().mimeType(CONTENT_TYPE).build();
    try (GcsOutputChannel out = service.createOrReplace(filename, options)) {
        IOUtils.copy(Channels.newChannel(new ByteArrayInputStream(bytes)), out);
    }

    BlobstoreService blobstoreService = BlobstoreServiceFactory.getBlobstoreService();
    BlobKey key =  blobstoreService.createGsBlobKey("/gs/GcsBucket/" + ts);
    byte[] fetched = blobstoreService.fetchData(key, 0, 10);
    Assert.assertArrayEquals(bytes, fetched);
}
 
Example #4
Source File: OauthRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public void copyObject(GcsFilename source, GcsFilename dest, GcsFileOptions fileOptions,
    long timeoutMillis) throws IOException {
  HTTPRequest req = makeRequest(dest, null, PUT, timeoutMillis);
  req.setHeader(new HTTPHeader(X_GOOG_COPY_SOURCE, makePath(source)));
  if (fileOptions != null) {
    req.setHeader(REPLACE_METADATA_HEADER);
    addOptionsHeaders(req, fileOptions);
  }
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  if (resp.getResponseCode() != 200) {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 
Example #5
Source File: OauthRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
private void addOptionsHeaders(HTTPRequest req, GcsFileOptions options) {
  if (options == null) {
    return;
  }
  if (options.getMimeType() != null) {
    req.setHeader(new HTTPHeader(CONTENT_TYPE, options.getMimeType()));
  }
  if (options.getAcl() != null) {
    req.setHeader(new HTTPHeader(ACL, options.getAcl()));
  }
  if (options.getCacheControl() != null) {
    req.setHeader(new HTTPHeader(CACHE_CONTROL, options.getCacheControl()));
  }
  if (options.getContentDisposition() != null) {
    req.setHeader(new HTTPHeader(CONTENT_DISPOSITION, options.getContentDisposition()));
  }
  if (options.getContentEncoding() != null) {
    req.setHeader(new HTTPHeader(CONTENT_ENCODING, options.getContentEncoding()));
  }
  for (Entry<String, String> entry : options.getUserMetadata().entrySet()) {
    req.setHeader(new HTTPHeader(X_GOOG_META + entry.getKey(), entry.getValue()));
  }
}
 
Example #6
Source File: OauthRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public RawGcsCreationToken beginObjectCreation(
    GcsFilename filename, GcsFileOptions options, long timeoutMillis) throws IOException {
  HTTPRequest req = makeRequest(filename, null, POST, timeoutMillis);
  req.setHeader(RESUMABLE_HEADER);
  addOptionsHeaders(req, options);
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  if (resp.getResponseCode() == 201) {
    String location = URLFetchUtils.getSingleHeader(resp, LOCATION);
    String queryString = new URL(location).getQuery();
    Preconditions.checkState(
        queryString != null, LOCATION + " header," + location + ", witout a query string");
    Map<String, String> params = Splitter.on('&').withKeyValueSeparator('=').split(queryString);
    Preconditions.checkState(params.containsKey(UPLOAD_ID),
        LOCATION + " header," + location + ", has a query string without " + UPLOAD_ID);
    return new GcsRestCreationToken(filename, params.get(UPLOAD_ID), 0);
  } else {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 
Example #7
Source File: RestoreCommitLogsActionTest.java    From nomulus with Apache License 2.0 6 votes vote down vote up
private Iterable<ImmutableObject> saveDiffFile(
    CommitLogCheckpoint checkpoint, ImmutableObject... entities) throws IOException {
  DateTime now = checkpoint.getCheckpointTime();
  List<ImmutableObject> allEntities = Lists.asList(checkpoint, entities);
  ByteArrayOutputStream output = new ByteArrayOutputStream();
  for (ImmutableObject entity : allEntities) {
    serializeEntity(entity, output);
  }
  gcsService.createOrReplace(
      new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now),
      new GcsFileOptions.Builder()
          .addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(1).toString())
          .build(),
      ByteBuffer.wrap(output.toByteArray()));
  return allEntities;
}
 
Example #8
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public void copyObject(GcsFilename source, GcsFilename dest, GcsFileOptions fileOptions,
    long timeoutMillis) throws IOException {
  ensureInitialized();
  GcsFileMetadata meta = getObjectMetadata(source, timeoutMillis);
  if (meta == null) {
    throw new FileNotFoundException(this + ": No such file: " + source);
  }
  if (fileOptions == null) {
    fileOptions = meta.getOptions();
  }

  Token token = beginObjectCreation(dest, fileOptions, timeoutMillis);
  appendFileContentsToToken(source, token);
  finishObjectCreation(token, ByteBuffer.allocate(0), timeoutMillis);
}
 
Example #9
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public void composeObject(Iterable<String> source, GcsFilename dest, long timeoutMillis)
    throws IOException {
  ensureInitialized();
  int size = Iterables.size(source);
  if (size > 32) {
    throw new IOException("Compose attempted with too many components. Limit is 32");
  }
  if (size < 2) {
    throw new IOException("You must provide at least two source components.");
  }
  Token token = beginObjectCreation(dest, GcsFileOptions.getDefaultInstance(), timeoutMillis);

  for (String filename : source) {
    GcsFilename sourceGcsFilename = new GcsFilename(dest.getBucketName(), filename);
    appendFileContentsToToken(sourceGcsFilename, token);
  }
  finishObjectCreation(token, ByteBuffer.allocate(0), timeoutMillis);
}
 
Example #10
Source File: GcsExampleServlet.java    From appengine-gcs-client with Apache License 2.0 5 votes vote down vote up
/**
   * Writes the payload of the incoming post as the contents of a file to GCS.
   * If the request path is /gcs/Foo/Bar this will be interpreted as
   * a request to create a GCS file named Bar in bucket Foo.
   */
//[START doPost]
  @Override
  public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException {
    GcsFileOptions instance = GcsFileOptions.getDefaultInstance();
    GcsFilename fileName = getFileName(req);
    GcsOutputChannel outputChannel;
    outputChannel = gcsService.createOrReplace(fileName, instance);
    copy(req.getInputStream(), Channels.newOutputStream(outputChannel));
  }
 
Example #11
Source File: GCSClientTest.java    From appengine-tck with Apache License 2.0 5 votes vote down vote up
@Test
@InSequence(4)
public void testOptionsAndMetadata() throws IOException {
    GcsFilename filename = new GcsFilename(bucket, OBJECT_NAME + "4");
    GcsFileOptions option = new GcsFileOptions.Builder()
        .acl("public-read")
        .cacheControl("Cache-Control: public, max-age=3600")
        .contentEncoding("Content-Encoding: gzip")
        .contentDisposition("Content-Disposition: attachment")
        .mimeType("text/html")
        .addUserMetadata("userKey", "UserMetadata")
        .build();

    GcsOutputChannel writeChannel = gcsService.createOrReplace(filename, option);
    try (PrintWriter out = new PrintWriter(Channels.newWriter(writeChannel, "UTF8"))) {
        out.println(CONTENT);
        out.flush();
    }

    GcsFileMetadata metaData = gcsService.getMetadata(filename);
    GcsFileOptions option2 = metaData.getOptions();
    try {
        assertEquals(filename, metaData.getFilename());
    } finally {
        gcsService.delete(filename);
    }

    assertEquals("Cache-Control: public, max-age=3600", option2.getCacheControl());
    assertEquals("Content-Encoding: gzip", option2.getContentEncoding());
    assertEquals("Content-Disposition: attachment", option2.getContentDisposition());
    assertEquals("text/html", option2.getMimeType());
    assertEquals("Content-Encoding: gzip", option2.getContentEncoding());
    Map<String, String> userMetadata = option2.getUserMetadata();
    assertEquals(1, userMetadata.size());
    String key = userMetadata.keySet().iterator().next();
    assertEquals("UserMetadata", userMetadata.get(key));
    assertEquals("public-read", option2.getAcl());
}
 
Example #12
Source File: OauthRawGcsService.java    From appengine-gcs-client with Apache License 2.0 5 votes vote down vote up
@Override
public void putObject(GcsFilename filename, GcsFileOptions options, ByteBuffer content,
    long timeoutMillis) throws IOException {
  HTTPRequest req = makeRequest(filename, null, PUT, timeoutMillis, content);
  addOptionsHeaders(req, options);
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  if (resp.getResponseCode() != 200) {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 
Example #13
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 5 votes vote down vote up
@Override
public void putObject(GcsFilename filename, GcsFileOptions options, ByteBuffer content,
    long timeoutMillis) throws IOException {
  ensureInitialized();
  Token token = beginObjectCreation(filename, options, timeoutMillis);
  finishObjectCreation(token, content, timeoutMillis);
}
 
Example #14
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 5 votes vote down vote up
private GcsFileMetadata createGcsFileMetadataFromBlobstore(Entity entity, GcsFilename filename) {
  return new GcsFileMetadata(
      filename,
      GcsFileOptions.getDefaultInstance(),
      "",
      (Long) entity.getProperty("size"),
      (Date) entity.getProperty("creation"));
}
 
Example #15
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 5 votes vote down vote up
@Override
public Token beginObjectCreation(GcsFilename filename, GcsFileOptions options, long timeoutMillis)
    throws IOException {
  ensureInitialized();
  inMemoryData.put(filename, new ArrayList<ByteBuffer>());
  return new Token(filename, options, 0);
}
 
Example #16
Source File: GalleryServiceImpl.java    From appinventor-extensions with Apache License 2.0 5 votes vote down vote up
/**
 * store aia file on cloud server
 * @param galleryId gallery id
 * @param projectId project id
 * @param projectName project name
 */
private void storeAIA(long galleryId, long projectId, String projectName) throws IOException {

  final String userId = userInfoProvider.getUserId();
  // build the aia file name using the ai project name and code stolen
  // from DownloadServlet to normalize...
  String aiaName = StringUtils.normalizeForFilename(projectName) + ".aia";
  // grab the data for the aia file using code from DownloadServlet
  RawFile aiaFile = null;
  byte[] aiaBytes= null;
  ProjectSourceZip zipFile = fileExporter.exportProjectSourceZip(userId,
    projectId, true, false, aiaName, false, false, false, true);
  aiaFile = zipFile.getRawFile();
  aiaBytes = aiaFile.getContent();
  LOG.log(Level.INFO, "aiaFile numBytes:"+aiaBytes.length);
  // now stick the aia file into the gcs

  //String galleryKey = GalleryApp.getSourceKey(galleryId);//String.valueOf(galleryId);
  GallerySettings settings = loadGallerySettings();
  String galleryKey = settings.getSourceKey(galleryId);
  // setup cloud
  GcsService gcsService = GcsServiceFactory.createGcsService();

  //GcsFilename filename = new GcsFilename(GalleryApp.GALLERYBUCKET, galleryKey);
  GcsFilename filename = new GcsFilename(settings.getBucket(), galleryKey);

  GcsFileOptions options = new GcsFileOptions.Builder().mimeType("application/zip")
    .acl("public-read").cacheControl("no-cache").addUserMetadata("title", aiaName).build();
  GcsOutputChannel writeChannel = gcsService.createOrReplace(filename, options);
  writeChannel.write(ByteBuffer.wrap(aiaBytes));

  // Now finalize
  writeChannel.close();
}
 
Example #17
Source File: GoogleCloudStorageHelper.java    From teammates with GNU General Public License v2.0 5 votes vote down vote up
/**
 * Writes a byte array {@code imageData} as image to the Google Cloud Storage,
 * with the {@code googleId} as the identifier name for the image.
 *
 * @return the {@link BlobKey} used as the image's identifier in Google Cloud Storage
 */
public static String writeImageDataToGcs(String googleId, byte[] imageData, String contentType) throws IOException {
    GcsFilename gcsFilename = new GcsFilename(Config.PRODUCTION_GCS_BUCKETNAME, googleId);
    try (GcsOutputChannel outputChannel =
            GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance())
            .createOrReplace(gcsFilename, new GcsFileOptions.Builder().mimeType(contentType).build())) {

        outputChannel.write(ByteBuffer.wrap(imageData));
    }

    return createBlobKey(googleId);
}
 
Example #18
Source File: GcsDiffFileListerTest.java    From nomulus with Apache License 2.0 5 votes vote down vote up
private void addGcsFile(int fileAge, int prevAge) throws IOException {
  gcsService.createOrReplace(
      new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(fileAge)),
      new GcsFileOptions.Builder()
          .addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(prevAge).toString())
          .build(),
      ByteBuffer.wrap(new byte[]{1, 2, 3}));
}
 
Example #19
Source File: GcsDiffFileListerTest.java    From nomulus with Apache License 2.0 5 votes vote down vote up
@Before
public void before() throws Exception {
  diffLister.gcsService = gcsService;
  diffLister.gcsBucket = GCS_BUCKET;
  diffLister.executor = newDirectExecutorService();
  for (int i = 0; i < 5; i++) {
    gcsService.createOrReplace(
        new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(i)),
        new GcsFileOptions.Builder()
            .addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(i + 1).toString())
            .build(),
        ByteBuffer.wrap(new byte[]{1, 2, 3}));
  }
  LoggerConfig.getConfig(GcsDiffFileLister.class).addHandler(logHandler);
}
 
Example #20
Source File: GcsUtils.java    From nomulus with Apache License 2.0 5 votes vote down vote up
/** Determines most appropriate {@link GcsFileOptions} based on filename extension. */
private static GcsFileOptions getOptions(GcsFilename filename) {
  GcsFileOptions.Builder builder = new GcsFileOptions.Builder().cacheControl("no-cache");
  MediaType mediaType = EXTENSIONS.get(getLast(Splitter.on('.').split(filename.getObjectName())));
  if (mediaType != null) {
    builder = builder.mimeType(mediaType.type());
  }
  return builder.build();
}
 
Example #21
Source File: ObjectifyStorageIo.java    From appinventor-extensions with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
void setGcsFileContent(String gcsPath, byte[] content) throws IOException {
  GcsOutputChannel outputChannel = gcsService.createOrReplace(
      new GcsFilename(GCS_BUCKET_NAME, gcsPath),
      GcsFileOptions.getDefaultInstance());
  outputChannel.write(ByteBuffer.wrap(content));
  outputChannel.close();
}
 
Example #22
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 4 votes vote down vote up
Token(GcsFilename filename, GcsFileOptions options, long offset) {
  this.options = options;
  this.filename = checkNotNull(filename, "Null filename");
  this.offset = offset;
}
 
Example #23
Source File: ImagesServlet.java    From java-docs-samples with Apache License 2.0 4 votes vote down vote up
@Override
public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {

  // [START original_image]
  // Read the image.jpg resource into a ByteBuffer.
  FileInputStream fileInputStream = new FileInputStream(new File("WEB-INF/image.jpg"));
  FileChannel fileChannel = fileInputStream.getChannel();
  ByteBuffer byteBuffer = ByteBuffer.allocate((int) fileChannel.size());
  fileChannel.read(byteBuffer);

  byte[] imageBytes = byteBuffer.array();

  // Write the original image to Cloud Storage
  gcsService.createOrReplace(
      new GcsFilename(bucket, "image.jpeg"),
      new GcsFileOptions.Builder().mimeType("image/jpeg").build(),
      ByteBuffer.wrap(imageBytes));
  // [END original_image]

  // [START resize]
  // Get an instance of the imagesService we can use to transform images.
  ImagesService imagesService = ImagesServiceFactory.getImagesService();

  // Make an image directly from a byte array, and transform it.
  Image image = ImagesServiceFactory.makeImage(imageBytes);
  Transform resize = ImagesServiceFactory.makeResize(100, 50);
  Image resizedImage = imagesService.applyTransform(resize, image);

  // Write the transformed image back to a Cloud Storage object.
  gcsService.createOrReplace(
      new GcsFilename(bucket, "resizedImage.jpeg"),
      new GcsFileOptions.Builder().mimeType("image/jpeg").build(),
      ByteBuffer.wrap(resizedImage.getImageData()));
  // [END resize]

  // [START rotate]
  // Make an image from a Cloud Storage object, and transform it.
  BlobstoreService blobstoreService = BlobstoreServiceFactory.getBlobstoreService();
  BlobKey blobKey = blobstoreService.createGsBlobKey("/gs/" + bucket + "/image.jpeg");
  Image blobImage = ImagesServiceFactory.makeImageFromBlob(blobKey);
  Transform rotate = ImagesServiceFactory.makeRotate(90);
  Image rotatedImage = imagesService.applyTransform(rotate, blobImage);

  // Write the transformed image back to a Cloud Storage object.
  gcsService.createOrReplace(
      new GcsFilename(bucket, "rotatedImage.jpeg"),
      new GcsFileOptions.Builder().mimeType("image/jpeg").build(),
      ByteBuffer.wrap(rotatedImage.getImageData()));
  // [END rotate]

  // [START servingUrl]
  // Create a fixed dedicated URL that points to the GCS hosted file
  ServingUrlOptions options =
      ServingUrlOptions.Builder.withGoogleStorageFileName("/gs/" + bucket + "/image.jpeg")
          .imageSize(150)
          .crop(true)
          .secureUrl(true);
  String url = imagesService.getServingUrl(options);
  // [END servingUrl]

  // Output some simple HTML to display the images we wrote to Cloud Storage
  // in the browser.
  PrintWriter out = resp.getWriter();
  out.println("<html><body>\n");
  out.println(
      "<img src='//storage.cloud.google.com/" + bucket + "/image.jpeg' alt='AppEngine logo' />");
  out.println(
      "<img src='//storage.cloud.google.com/"
          + bucket
          + "/resizedImage.jpeg' alt='AppEngine logo resized' />");
  out.println(
      "<img src='//storage.cloud.google.com/"
          + bucket
          + "/rotatedImage.jpeg' alt='AppEngine logo rotated' />");
  out.println("<img src='" + url + "' alt='Hosted logo' />");
  out.println("</body></html>\n");
}
 
Example #24
Source File: GcsTestingUtils.java    From nomulus with Apache License 2.0 4 votes vote down vote up
/** Writes a Cloud Storage file. */
public static void writeGcsFile(GcsService gcsService, GcsFilename file, byte[] data)
    throws IOException {
  gcsService.createOrReplace(file, GcsFileOptions.getDefaultInstance(), ByteBuffer.wrap(data));
}
 
Example #25
Source File: ExportCommitLogDiffAction.java    From nomulus with Apache License 2.0 4 votes vote down vote up
@Override
public void run() {
  logger.atInfo().log(
      "Exporting commit log diffs between %s and %s.", lowerCheckpointTime, upperCheckpointTime);
  checkArgument(isAtOrAfter(lowerCheckpointTime, START_OF_TIME));
  checkArgument(lowerCheckpointTime.isBefore(upperCheckpointTime));
  // Load the boundary checkpoints - lower is exclusive and may not exist (on the first export,
  // when lowerCheckpointTime is START_OF_TIME), whereas the upper is inclusive and must exist.
  CommitLogCheckpoint lowerCheckpoint = lowerCheckpointTime.isAfter(START_OF_TIME)
      ? verifyNotNull(ofy().load().key(CommitLogCheckpoint.createKey(lowerCheckpointTime)).now())
      : null;
  CommitLogCheckpoint upperCheckpoint =
      verifyNotNull(ofy().load().key(CommitLogCheckpoint.createKey(upperCheckpointTime)).now());

  // Load the keys of all the manifests to include in this diff.
  List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
  logger.atInfo().log("Found %d manifests to export", sortedKeys.size());
  // Open an output channel to GCS, wrapped in a stream for convenience.
  try (OutputStream gcsStream = newOutputStream(gcsService.createOrReplace(
      new GcsFilename(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
      new GcsFileOptions.Builder()
          .addUserMetadata(LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString())
          .addUserMetadata(UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString())
          .addUserMetadata(NUM_TRANSACTIONS, Integer.toString(sortedKeys.size()))
          .build()))) {
    // Export the upper checkpoint itself.
    serializeEntity(upperCheckpoint, gcsStream);
    // If there are no manifests to export, stop early, now that we've written out the file with
    // the checkpoint itself (which is needed for restores, even if it's empty).
    if (sortedKeys.isEmpty()) {
      return;
    }
    // Export to GCS in chunks, one per fixed batch of commit logs.  While processing one batch,
    // asynchronously load the entities for the next one.
    List<List<Key<CommitLogManifest>>> keyChunks = partition(sortedKeys, batchSize);
    // Objectify's map return type is asynchronous. Calling .values() will block until it loads.
    Map<?, CommitLogManifest> nextChunkToExport = ofy().load().keys(keyChunks.get(0));
    for (int i = 0; i < keyChunks.size(); i++) {
      // Force the async load to finish.
      Collection<CommitLogManifest> chunkValues = nextChunkToExport.values();
      logger.atInfo().log("Loaded %d manifests", chunkValues.size());
      // Since there is no hard bound on how much data this might be, take care not to let the
      // Objectify session cache fill up and potentially run out of memory. This is the only safe
      // point to do this since at this point there is no async load in progress.
      ofy().clearSessionCache();
      // Kick off the next async load, which can happen in parallel to the current GCS export.
      if (i + 1 < keyChunks.size()) {
        nextChunkToExport = ofy().load().keys(keyChunks.get(i + 1));
      }
      exportChunk(gcsStream, chunkValues);
      logger.atInfo().log("Exported %d manifests", chunkValues.size());
    }
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  logger.atInfo().log("Exported %d manifests in total", sortedKeys.size());
}
 
Example #26
Source File: GalleryServlet.java    From appinventor-extensions with Apache License 2.0 4 votes vote down vote up
@Override
  public void doPost(HttpServletRequest req, HttpServletResponse resp) {
    setDefaultHeader(resp);
    UploadResponse uploadResponse;

    String uri = req.getRequestURI();
    // First, call split with no limit parameter.
    String[] uriComponents = uri.split("/");

    if (true) {
      String requestType = uriComponents[REQUEST_TYPE_INDEX];
      if (DEBUG) {
        LOG.info("######### GOT IN URI");
        LOG.info(requestType);
      }

      long project_Id = -1;
      String user_Id = "-1";
      if (requestType.equalsIgnoreCase("apps")) {
        project_Id = Long.parseLong(uriComponents[GALLERY_OR_USER_ID_INDEX]);
      } else if (requestType.equalsIgnoreCase("user")) {
        //the code below doesn't check if user_Id is the id of current user
        //user_Id = uriComponents[GALLERY_OR_USER_ID_INDEX];
        user_Id = userInfoProvider.getUserId();
      }
      InputStream uploadedStream;
      try {
        if(req.getContentLength() < MAX_IMAGE_FILE_SIZE){
          uploadedStream = getRequestStream(req, ServerLayout.UPLOAD_FILE_FORM_ELEMENT);

          // Converts the input stream to byte array
          byte[] buffer = new byte[8000];
          int bytesRead = 0;
          ByteArrayOutputStream bao = new ByteArrayOutputStream();
          while ((bytesRead = uploadedStream.read(buffer)) != -1) {
            bao.write(buffer, 0, bytesRead);
          }
          // Set up the cloud file (options)
          String key = "";
          GallerySettings settings = galleryService.loadGallerySettings();
          if (requestType.equalsIgnoreCase("apps")) {
            key = settings.getProjectImageKey(project_Id);
          } else if (requestType.equalsIgnoreCase("user")) {
            key =  settings.getUserImageKey(user_Id);
          }

          // setup cloud
          GcsService gcsService = GcsServiceFactory.createGcsService();
          GcsFilename filename = new GcsFilename(settings.getBucket(), key);
          GcsFileOptions options = new GcsFileOptions.Builder().mimeType("image/jpeg")
                  .acl("public-read").cacheControl("no-cache").build();
          GcsOutputChannel writeChannel = gcsService.createOrReplace(filename, options);
          writeChannel.write(ByteBuffer.wrap(bao.toByteArray()));

          // Now finalize
          writeChannel.close();

          uploadResponse = new UploadResponse(UploadResponse.Status.SUCCESS);
        }else{
          /*file exceeds size of MAX_IMAGE_FILE_SIZE*/
          uploadResponse = new UploadResponse(UploadResponse.Status.FILE_TOO_LARGE);
        }

        // Now, get the PrintWriter for the servlet response and print the UploadResponse.
        // On the client side, in the onSubmitComplete method in ode/client/utils/Uploader.java, the
        // UploadResponse value will be retrieved as a String via the
        // FormSubmitCompleteEvent.getResults() method.
        PrintWriter out = resp.getWriter();
        out.print(uploadResponse.formatAsHtml());

      } catch (Exception e) {
        throw CrashReport.createAndLogError(LOG, req, null, e);
      }
      // Set http response information
      resp.setStatus(HttpServletResponse.SC_OK);
    }

    // Now, get the PrintWriter for the servlet response and print the UploadResponse.
    // On the client side, in the onSubmitComplete method in ode/client/utils/Uploader.java, the
    // UploadResponse value will be retrieved as a String via the
    // FormSubmitCompleteEvent.getResults() method.
//    PrintWriter out = resp.getWriter();
//    out.print(uploadResponse.formatAsHtml());

    // Set http response information
    resp.setStatus(HttpServletResponse.SC_OK);
  }
 
Example #27
Source File: GalleryServiceImpl.java    From appinventor-extensions with Apache License 2.0 4 votes vote down vote up
/**
 * when an app is published/updated, we need to move the image
 * that was temporarily uploaded into projects/projectid/image
 * into the gallery image
 * @param app gallery app
 */
private void setGalleryAppImage(GalleryApp app) {
  // best thing would be if GCS has a mv op, we can just do that.
  // don't think that is there, though, so for now read one and write to other
  // First, read the file from projects name
  boolean lockForRead = false;
  //String projectImageKey = app.getProjectImageKey();
  GallerySettings settings = loadGallerySettings();
  String projectImageKey = settings.getProjectImageKey(app.getProjectId());
  try {
    GcsService gcsService = GcsServiceFactory.createGcsService();
    //GcsFilename filename = new GcsFilename(GalleryApp.GALLERYBUCKET, projectImageKey);
    GcsFilename filename = new GcsFilename(settings.getBucket(), projectImageKey);
    GcsInputChannel readChannel = gcsService.openReadChannel(filename, 0);
    InputStream gcsis = Channels.newInputStream(readChannel);

    byte[] buffer = new byte[8000];
    int bytesRead = 0;
    ByteArrayOutputStream bao = new ByteArrayOutputStream();

    while ((bytesRead = gcsis.read(buffer)) != -1) {
      bao.write(buffer, 0, bytesRead);
    }
    // close the project image file
    readChannel.close();

    // if image is greater than 200 X 200, it will be scaled (200 X 200).
    // otherwise, it will be stored as origin.
    byte[] oldImageData = bao.toByteArray();
    byte[] newImageData;
    ImagesService imagesService = ImagesServiceFactory.getImagesService();
    Image oldImage = ImagesServiceFactory.makeImage(oldImageData);
    //if image size is too big, scale it to a smaller size.
    if(oldImage.getWidth() > 200 && oldImage.getHeight() > 200){
        Transform resize = ImagesServiceFactory.makeResize(200, 200);
        Image newImage = imagesService.applyTransform(resize, oldImage);
        newImageData = newImage.getImageData();
    }else{
        newImageData = oldImageData;
    }

    // set up the cloud file (options)
    // After publish, copy the /projects/projectId image into /apps/appId
    //String galleryKey = app.getImageKey();
    String galleryKey = settings.getImageKey(app.getGalleryAppId());

    //GcsFilename outfilename = new GcsFilename(GalleryApp.GALLERYBUCKET, galleryKey);
    GcsFilename outfilename = new GcsFilename(settings.getBucket(), galleryKey);
    GcsFileOptions options = new GcsFileOptions.Builder().mimeType("image/jpeg")
        .acl("public-read").cacheControl("no-cache").build();
    GcsOutputChannel writeChannel = gcsService.createOrReplace(outfilename, options);
    writeChannel.write(ByteBuffer.wrap(newImageData));

    // Now finalize
    writeChannel.close();

  } catch (IOException e) {
    // TODO Auto-generated catch block
    LOG.log(Level.INFO, "FAILED WRITING IMAGE TO GCS");
    e.printStackTrace();
  }
}