com.google.appengine.tools.cloudstorage.GcsService Java Examples

/**
 * delete aia file based on given gallery id
 * @param galleryId gallery id
 */
private void deleteAIA(long galleryId) {
  try {
    //String galleryKey = GalleryApp.getSourceKey(galleryId);
    GallerySettings settings = loadGallerySettings();
    String galleryKey = settings.getSourceKey(galleryId);

    // setup cloud
    GcsService gcsService = GcsServiceFactory.createGcsService();
    //GcsFilename filename = new GcsFilename(GalleryApp.GALLERYBUCKET, galleryKey);
    GcsFilename filename = new GcsFilename(settings.getBucket(), galleryKey);
    gcsService.delete(filename);
  } catch (IOException e) {
    // TODO Auto-generated catch block
    LOG.log(Level.INFO, "FAILED GCS delete");
    e.printStackTrace();
  }
}
 

private void deleteImage(long galleryId) {
  try {
    //String galleryKey = GalleryApp.getImageKey(galleryId);
    GallerySettings settings = loadGallerySettings();
    String galleryKey = settings.getSourceKey(galleryId);
    // setup cloud
    GcsService gcsService = GcsServiceFactory.createGcsService();
    //GcsFilename filename = new GcsFilename(GalleryApp.GALLERYBUCKET, galleryKey);
    GcsFilename filename = new GcsFilename(settings.getBucket(), galleryKey);
    gcsService.delete(filename);
  } catch (IOException e) {
    // TODO Auto-generated catch block
    LOG.log(Level.INFO, "FAILED GCS delete");
    e.printStackTrace();
  }
}
 

@Test
@InSequence(100)
public void testCreateGsBlobKey() throws Exception {
    final long ts = System.currentTimeMillis();
    final byte[] bytes = "FooBar".getBytes();

    GcsService service = GcsServiceFactory.createGcsService();
    GcsFilename filename = new GcsFilename("GcsBucket", String.valueOf(ts));
    GcsFileOptions options = new GcsFileOptions.Builder().mimeType(CONTENT_TYPE).build();
    try (GcsOutputChannel out = service.createOrReplace(filename, options)) {
        IOUtils.copy(Channels.newChannel(new ByteArrayInputStream(bytes)), out);
    }

    BlobstoreService blobstoreService = BlobstoreServiceFactory.getBlobstoreService();
    BlobKey key =  blobstoreService.createGsBlobKey("/gs/GcsBucket/" + ts);
    byte[] fetched = blobstoreService.fetchData(key, 0, 10);
    Assert.assertArrayEquals(bytes, fetched);
}
 

/**
 * store aia file on cloud server
 * @param galleryId gallery id
 * @param projectId project id
 * @param projectName project name
 */
private void storeAIA(long galleryId, long projectId, String projectName) throws IOException {

  final String userId = userInfoProvider.getUserId();
  // build the aia file name using the ai project name and code stolen
  // from DownloadServlet to normalize...
  String aiaName = StringUtils.normalizeForFilename(projectName) + ".aia";
  // grab the data for the aia file using code from DownloadServlet
  RawFile aiaFile = null;
  byte[] aiaBytes= null;
  ProjectSourceZip zipFile = fileExporter.exportProjectSourceZip(userId,
    projectId, true, false, aiaName, false, false, false, true);
  aiaFile = zipFile.getRawFile();
  aiaBytes = aiaFile.getContent();
  LOG.log(Level.INFO, "aiaFile numBytes:"+aiaBytes.length);
  // now stick the aia file into the gcs

  //String galleryKey = GalleryApp.getSourceKey(galleryId);//String.valueOf(galleryId);
  GallerySettings settings = loadGallerySettings();
  String galleryKey = settings.getSourceKey(galleryId);
  // setup cloud
  GcsService gcsService = GcsServiceFactory.createGcsService();

  //GcsFilename filename = new GcsFilename(GalleryApp.GALLERYBUCKET, galleryKey);
  GcsFilename filename = new GcsFilename(settings.getBucket(), galleryKey);

  GcsFileOptions options = new GcsFileOptions.Builder().mimeType("application/zip")
    .acl("public-read").cacheControl("no-cache").addUserMetadata("title", aiaName).build();
  GcsOutputChannel writeChannel = gcsService.createOrReplace(filename, options);
  writeChannel.write(ByteBuffer.wrap(aiaBytes));

  // Now finalize
  writeChannel.close();
}
 

/** Slurps a Cloud Storage file into memory. */
public static byte[] readGcsFile(GcsService gcsService, GcsFilename file)
    throws IOException {
  try (InputStream input = Channels.newInputStream(gcsService.openReadChannel(file, 0))) {
    return ByteStreams.toByteArray(input);
  }
}
 

/**
 * when an app is published/updated, we need to move the image
 * that was temporarily uploaded into projects/projectid/image
 * into the gallery image
 * @param app gallery app
 */
private void setGalleryAppImage(GalleryApp app) {
  // best thing would be if GCS has a mv op, we can just do that.
  // don't think that is there, though, so for now read one and write to other
  // First, read the file from projects name
  boolean lockForRead = false;
  //String projectImageKey = app.getProjectImageKey();
  GallerySettings settings = loadGallerySettings();
  String projectImageKey = settings.getProjectImageKey(app.getProjectId());
  try {
    GcsService gcsService = GcsServiceFactory.createGcsService();
    //GcsFilename filename = new GcsFilename(GalleryApp.GALLERYBUCKET, projectImageKey);
    GcsFilename filename = new GcsFilename(settings.getBucket(), projectImageKey);
    GcsInputChannel readChannel = gcsService.openReadChannel(filename, 0);
    InputStream gcsis = Channels.newInputStream(readChannel);

    byte[] buffer = new byte[8000];
    int bytesRead = 0;
    ByteArrayOutputStream bao = new ByteArrayOutputStream();

    while ((bytesRead = gcsis.read(buffer)) != -1) {
      bao.write(buffer, 0, bytesRead);
    }
    // close the project image file
    readChannel.close();

    // if image is greater than 200 X 200, it will be scaled (200 X 200).
    // otherwise, it will be stored as origin.
    byte[] oldImageData = bao.toByteArray();
    byte[] newImageData;
    ImagesService imagesService = ImagesServiceFactory.getImagesService();
    Image oldImage = ImagesServiceFactory.makeImage(oldImageData);
    //if image size is too big, scale it to a smaller size.
    if(oldImage.getWidth() > 200 && oldImage.getHeight() > 200){
        Transform resize = ImagesServiceFactory.makeResize(200, 200);
        Image newImage = imagesService.applyTransform(resize, oldImage);
        newImageData = newImage.getImageData();
    }else{
        newImageData = oldImageData;
    }

    // set up the cloud file (options)
    // After publish, copy the /projects/projectId image into /apps/appId
    //String galleryKey = app.getImageKey();
    String galleryKey = settings.getImageKey(app.getGalleryAppId());

    //GcsFilename outfilename = new GcsFilename(GalleryApp.GALLERYBUCKET, galleryKey);
    GcsFilename outfilename = new GcsFilename(settings.getBucket(), galleryKey);
    GcsFileOptions options = new GcsFileOptions.Builder().mimeType("image/jpeg")
        .acl("public-read").cacheControl("no-cache").build();
    GcsOutputChannel writeChannel = gcsService.createOrReplace(outfilename, options);
    writeChannel.write(ByteBuffer.wrap(newImageData));

    // Now finalize
    writeChannel.close();

  } catch (IOException e) {
    // TODO Auto-generated catch block
    LOG.log(Level.INFO, "FAILED WRITING IMAGE TO GCS");
    e.printStackTrace();
  }
}
 

@Override
  public void doPost(HttpServletRequest req, HttpServletResponse resp) {
    setDefaultHeader(resp);
    UploadResponse uploadResponse;

    String uri = req.getRequestURI();
    // First, call split with no limit parameter.
    String[] uriComponents = uri.split("/");

    if (true) {
      String requestType = uriComponents[REQUEST_TYPE_INDEX];
      if (DEBUG) {
        LOG.info("######### GOT IN URI");
        LOG.info(requestType);
      }

      long project_Id = -1;
      String user_Id = "-1";
      if (requestType.equalsIgnoreCase("apps")) {
        project_Id = Long.parseLong(uriComponents[GALLERY_OR_USER_ID_INDEX]);
      } else if (requestType.equalsIgnoreCase("user")) {
        //the code below doesn't check if user_Id is the id of current user
        //user_Id = uriComponents[GALLERY_OR_USER_ID_INDEX];
        user_Id = userInfoProvider.getUserId();
      }
      InputStream uploadedStream;
      try {
        if(req.getContentLength() < MAX_IMAGE_FILE_SIZE){
          uploadedStream = getRequestStream(req, ServerLayout.UPLOAD_FILE_FORM_ELEMENT);

          // Converts the input stream to byte array
          byte[] buffer = new byte[8000];
          int bytesRead = 0;
          ByteArrayOutputStream bao = new ByteArrayOutputStream();
          while ((bytesRead = uploadedStream.read(buffer)) != -1) {
            bao.write(buffer, 0, bytesRead);
          }
          // Set up the cloud file (options)
          String key = "";
          GallerySettings settings = galleryService.loadGallerySettings();
          if (requestType.equalsIgnoreCase("apps")) {
            key = settings.getProjectImageKey(project_Id);
          } else if (requestType.equalsIgnoreCase("user")) {
            key =  settings.getUserImageKey(user_Id);
          }

          // setup cloud
          GcsService gcsService = GcsServiceFactory.createGcsService();
          GcsFilename filename = new GcsFilename(settings.getBucket(), key);
          GcsFileOptions options = new GcsFileOptions.Builder().mimeType("image/jpeg")
                  .acl("public-read").cacheControl("no-cache").build();
          GcsOutputChannel writeChannel = gcsService.createOrReplace(filename, options);
          writeChannel.write(ByteBuffer.wrap(bao.toByteArray()));

          // Now finalize
          writeChannel.close();

          uploadResponse = new UploadResponse(UploadResponse.Status.SUCCESS);
        }else{
          /*file exceeds size of MAX_IMAGE_FILE_SIZE*/
          uploadResponse = new UploadResponse(UploadResponse.Status.FILE_TOO_LARGE);
        }

        // Now, get the PrintWriter for the servlet response and print the UploadResponse.
        // On the client side, in the onSubmitComplete method in ode/client/utils/Uploader.java, the
        // UploadResponse value will be retrieved as a String via the
        // FormSubmitCompleteEvent.getResults() method.
        PrintWriter out = resp.getWriter();
        out.print(uploadResponse.formatAsHtml());

      } catch (Exception e) {
        throw CrashReport.createAndLogError(LOG, req, null, e);
      }
      // Set http response information
      resp.setStatus(HttpServletResponse.SC_OK);
    }

    // Now, get the PrintWriter for the servlet response and print the UploadResponse.
    // On the client side, in the onSubmitComplete method in ode/client/utils/Uploader.java, the
    // UploadResponse value will be retrieved as a String via the
    // FormSubmitCompleteEvent.getResults() method.
//    PrintWriter out = resp.getWriter();
//    out.print(uploadResponse.formatAsHtml());

    // Set http response information
    resp.setStatus(HttpServletResponse.SC_OK);
  }
 

@Provides
static GcsService provideGcsService() {
  return gcsService;
}
 

@Inject
public GcsUtils(GcsService gcsService, @Config("gcsBufferSize") int bufferSize) {
  this.gcsService = gcsService;
  this.bufferSize = bufferSize;
}
 

@Test
public void testList_patchesHoles() {
  // Fake out the GCS list() method to return only the first and last file.
  // We can't use Mockito.spy() because GcsService's impl is final.
  diffLister.gcsService = (GcsService) newProxyInstance(
      GcsService.class.getClassLoader(),
      new Class<?>[] {GcsService.class},
      new InvocationHandler() {
        @Override
        public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
          if (method.getName().equals("list")) {
            // ListResult is an incredibly annoying thing to construct. It needs to be fed from a
            // Callable that returns Iterators, each representing a batch of results.
            return new ListResult(new Callable<Iterator<ListItem>>() {
              boolean called = false;

              @Override
              public Iterator<ListItem> call() {
                try {
                  return called ? null : Iterators.forArray(
                      new ListItem.Builder()
                          .setName(DIFF_FILE_PREFIX + now)
                          .build(),
                      new ListItem.Builder()
                          .setName(DIFF_FILE_PREFIX + now.minusMinutes(4))
                          .build());
                } finally {
                  called = true;
                }
              }});
          }
          return method.invoke(gcsService, args);
        }});
  DateTime fromTime = now.minusMinutes(4).minusSeconds(1);
  // Request all files with checkpoint > fromTime.
  assertThat(listDiffFiles(fromTime, null))
      .containsExactly(
          now.minusMinutes(4),
          now.minusMinutes(3),
          now.minusMinutes(2),
          now.minusMinutes(1),
          now)
      .inOrder();
}
 

/** Writes a Cloud Storage file. */
public static void writeGcsFile(GcsService gcsService, GcsFilename file, byte[] data)
    throws IOException {
  gcsService.createOrReplace(file, GcsFileOptions.getDefaultInstance(), ByteBuffer.wrap(data));
}
 

@Override
protected void migrateEntity(Account oldAccount) throws Exception {
    String oldGoogleId = oldAccount.getGoogleId();
    String newGoogleId = generateNewGoogleId(oldAccount);

    Key<Account> oldAccountKey = Key.create(Account.class, oldAccount.getGoogleId());
    Key<StudentProfile> oldStudentProfileKey = Key.create(oldAccountKey, StudentProfile.class, oldGoogleId);
    StudentProfile oldStudentProfile = ofy().load().key(oldStudentProfileKey).now();

    List<CourseStudent> oldStudents = ofy().load().type(CourseStudent.class)
            .filter("googleId =", oldGoogleId).list();

    List<Instructor> oldInstructors = ofy().load().type(Instructor.class)
            .filter("googleId =", oldGoogleId).list();

    // update students and instructors

    if (!oldStudents.isEmpty()) {
        oldStudents.forEach(student -> student.setGoogleId(newGoogleId));
        ofy().save().entities(oldStudents).now();
    }

    if (!oldInstructors.isEmpty()) {
        oldInstructors.forEach(instructor -> instructor.setGoogleId(newGoogleId));
        ofy().save().entities(oldInstructors).now();
        instructorsDb.putDocuments(
                oldInstructors.stream().map(InstructorAttributes::valueOf).collect(Collectors.toList()));
    }

    // recreate account and student profile

    oldAccount.setGoogleId(newGoogleId);
    if (ofy().load().type(Account.class).id(newGoogleId).now() == null) {
        ofy().save().entity(oldAccount).now();
    } else {
        log(String.format("Skip creation of new account as account (%s) already exists", newGoogleId));
    }
    ofy().delete().type(Account.class).id(oldGoogleId).now();

    if (oldStudentProfile != null) {
        String pictureKey = oldStudentProfile.getPictureKey();

        if (!ClientProperties.isTargetUrlDevServer()) {
            try {
                GcsFilename oldGcsFilename = new GcsFilename(Config.PRODUCTION_GCS_BUCKETNAME, oldGoogleId);
                GcsFilename newGcsFilename = new GcsFilename(Config.PRODUCTION_GCS_BUCKETNAME, newGoogleId);
                GcsService gcsService = GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
                gcsService.copy(oldGcsFilename, newGcsFilename);
                gcsService.delete(oldGcsFilename);
                pictureKey = GoogleCloudStorageHelper.createBlobKey(newGoogleId);
            } catch (Exception e) {
                log("Profile picture not exist or error during copy: " + e.getMessage());
            }
        }

        oldStudentProfile.setGoogleId(newGoogleId);
        oldStudentProfile.setPictureKey(pictureKey);
        ofy().save().entity(oldStudentProfile).now();
        ofy().delete().key(oldStudentProfileKey).now();
    }

    log(String.format("Complete migration for account with googleId %s. The new googleId is %s",
            oldGoogleId, newGoogleId));
}