com.google.appengine.tools.cloudstorage.RetryParams Java Examples

ObjectifyStorageIo() {
  RetryParams retryParams = new RetryParams.Builder().initialRetryDelayMillis(100)
    .retryMaxAttempts(10)
    .totalRetryPeriodMillis(10000).build();
  if (DEBUG) {
    LOG.log(Level.INFO, "RetryParams: getInitialRetryDelayMillis() = " + retryParams.getInitialRetryDelayMillis());
    LOG.log(Level.INFO, "RetryParams: getRequestTimeoutMillis() = " + retryParams.getRequestTimeoutMillis());
    LOG.log(Level.INFO, "RetryParams: getRetryDelayBackoffFactor() = " + retryParams.getRetryDelayBackoffFactor());
    LOG.log(Level.INFO, "RetryParams: getRetryMaxAttempts() = " + retryParams.getRetryMaxAttempts());
    LOG.log(Level.INFO, "RetryParams: getRetryMinAttempts() = " + retryParams.getRetryMinAttempts());
    LOG.log(Level.INFO, "RetryParams: getTotalRetryPeriodMillis() = " + retryParams.getTotalRetryPeriodMillis());
  }
  gcsService = GcsServiceFactory.createGcsService(retryParams);
  memcache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
  initMotd();
}
 

@Override
public void reduce(String tld, ReducerInput<String> stanzas) {
  String stanzaCounter = tld + " stanzas";
  GcsFilename filename =
      new GcsFilename(bucket, String.format(FILENAME_FORMAT, tld, exportTime));
  GcsUtils cloudStorage =
      new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
  try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
      Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8);
      PrintWriter writer = new PrintWriter(osWriter)) {
    writer.printf(HEADER_FORMAT, tld);
    for (Iterator<String> stanzaIter = filter(stanzas, Objects::nonNull);
        stanzaIter.hasNext(); ) {
      writer.println(stanzaIter.next());
      getContext().incrementCounter(stanzaCounter);
    }
    writer.flush();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

private void exportToGcs(String tld, String domains) {
  GcsFilename filename = new GcsFilename(gcsBucket, tld + ".txt");
  GcsUtils cloudStorage =
      new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
  try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
      Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) {
    osWriter.write(domains);
  } catch (IOException e) {
    logger.atSevere().withCause(e).log(
        "Error exporting registered domains for TLD %s to GCS.", tld);
  }
  getContext().incrementCounter("domain lists written out to GCS");
}
 

private TaskOptions toTaskOptions(Task task) {
  final QueueSettings queueSettings = task.getQueueSettings();

  TaskOptions taskOptions = TaskOptions.Builder.withUrl(TaskHandler.handleTaskUrl());
  if (queueSettings.getOnBackend() != null) {
    taskOptions.header("Host", BackendServiceFactory.getBackendService().getBackendAddress(
        queueSettings.getOnBackend()));
  } else {

    String versionHostname = RetryHelper.runWithRetries(new Callable<String>() {
      @Override
      public String call() {
        ModulesService service = ModulesServiceFactory.getModulesService();
        String module = queueSettings.getOnModule();
        String version = queueSettings.getModuleVersion();
        if (module == null) {
          module = service.getCurrentModule();
          version = service.getCurrentVersion();
        }
        return service.getVersionHostname(module, version);
      }
    }, RetryParams.getDefaultInstance(), MODULES_EXCEPTION_HANDLER);
    taskOptions.header("Host", versionHostname);
  }

  Long delayInSeconds = queueSettings.getDelayInSeconds();
  if (null != delayInSeconds) {
    taskOptions.countdownMillis(delayInSeconds * 1000L);
    queueSettings.setDelayInSeconds(null);
  }
  addProperties(taskOptions, task.toProperties());
  String taskName = task.getName();
  if (null != taskName) {
    taskOptions.taskName(taskName);
  }
  return taskOptions;
}
 

/**
 * Writes a byte array {@code imageData} as image to the Google Cloud Storage,
 * with the {@code googleId} as the identifier name for the image.
 *
 * @return the {@link BlobKey} used as the image's identifier in Google Cloud Storage
 */
public static String writeImageDataToGcs(String googleId, byte[] imageData, String contentType) throws IOException {
    GcsFilename gcsFilename = new GcsFilename(Config.PRODUCTION_GCS_BUCKETNAME, googleId);
    try (GcsOutputChannel outputChannel =
            GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance())
            .createOrReplace(gcsFilename, new GcsFileOptions.Builder().mimeType(contentType).build())) {

        outputChannel.write(ByteBuffer.wrap(imageData));
    }

    return createBlobKey(googleId);
}
 

@Override
protected void migrateEntity(Account oldAccount) throws Exception {
    String oldGoogleId = oldAccount.getGoogleId();
    String newGoogleId = generateNewGoogleId(oldAccount);

    Key<Account> oldAccountKey = Key.create(Account.class, oldAccount.getGoogleId());
    Key<StudentProfile> oldStudentProfileKey = Key.create(oldAccountKey, StudentProfile.class, oldGoogleId);
    StudentProfile oldStudentProfile = ofy().load().key(oldStudentProfileKey).now();

    List<CourseStudent> oldStudents = ofy().load().type(CourseStudent.class)
            .filter("googleId =", oldGoogleId).list();

    List<Instructor> oldInstructors = ofy().load().type(Instructor.class)
            .filter("googleId =", oldGoogleId).list();

    // update students and instructors

    if (!oldStudents.isEmpty()) {
        oldStudents.forEach(student -> student.setGoogleId(newGoogleId));
        ofy().save().entities(oldStudents).now();
    }

    if (!oldInstructors.isEmpty()) {
        oldInstructors.forEach(instructor -> instructor.setGoogleId(newGoogleId));
        ofy().save().entities(oldInstructors).now();
        instructorsDb.putDocuments(
                oldInstructors.stream().map(InstructorAttributes::valueOf).collect(Collectors.toList()));
    }

    // recreate account and student profile

    oldAccount.setGoogleId(newGoogleId);
    if (ofy().load().type(Account.class).id(newGoogleId).now() == null) {
        ofy().save().entity(oldAccount).now();
    } else {
        log(String.format("Skip creation of new account as account (%s) already exists", newGoogleId));
    }
    ofy().delete().type(Account.class).id(oldGoogleId).now();

    if (oldStudentProfile != null) {
        String pictureKey = oldStudentProfile.getPictureKey();

        if (!ClientProperties.isTargetUrlDevServer()) {
            try {
                GcsFilename oldGcsFilename = new GcsFilename(Config.PRODUCTION_GCS_BUCKETNAME, oldGoogleId);
                GcsFilename newGcsFilename = new GcsFilename(Config.PRODUCTION_GCS_BUCKETNAME, newGoogleId);
                GcsService gcsService = GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
                gcsService.copy(oldGcsFilename, newGcsFilename);
                gcsService.delete(oldGcsFilename);
                pictureKey = GoogleCloudStorageHelper.createBlobKey(newGoogleId);
            } catch (Exception e) {
                log("Profile picture not exist or error during copy: " + e.getMessage());
            }
        }

        oldStudentProfile.setGoogleId(newGoogleId);
        oldStudentProfile.setPictureKey(pictureKey);
        ofy().save().entity(oldStudentProfile).now();
        ofy().delete().key(oldStudentProfileKey).now();
    }

    log(String.format("Complete migration for account with googleId %s. The new googleId is %s",
            oldGoogleId, newGoogleId));
}