org.apache.reef.runtime.local.client.LocalRuntimeConfiguration Java Examples

/**
 * Get deploy mode configuration.
 * @param jobConf job configuration to get deploy mode.
 * @return deploy mode configuration.
 * @throws InjectionException exception while injection.
 */
public static Configuration getDeployModeConf(final Configuration jobConf) throws InjectionException {
  final Injector injector = TANG.newInjector(jobConf);
  final String deployMode = injector.getNamedInstance(JobConf.DeployMode.class);
  switch (deployMode) {
    case "local":
      return LocalRuntimeConfiguration.CONF
          .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, LOCAL_NUMBER_OF_EVALUATORS)
          .build();
    case "yarn":
      return YarnClientConfiguration.CONF
          .set(YarnClientConfiguration.JVM_HEAP_SLACK, injector.getNamedInstance(JobConf.JVMHeapSlack.class))
          .build();
    default:
      throw new UnsupportedOperationException(deployMode);
  }
}
 

/**
 * @return (immutable) TANG Configuration object.
 * @throws BindException      if configuration injector fails.
 * @throws InjectionException if the Local.class parameter is not injected.
 */
private static Configuration getRunTimeConfiguration(final boolean isLocal) throws BindException {

  final Configuration runtimeConfiguration;

  if (isLocal) {
    LOG.log(Level.INFO, "Running Failure demo on the local runtime");
    runtimeConfiguration = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
        .build();
  } else {
    LOG.log(Level.INFO, "Running Failure demo on YARN");
    runtimeConfiguration = YarnClientConfiguration.CONF.build();
  }

  return runtimeConfiguration;
}
 

/**
 * @return the runtime configuration, based on the parameters passed from C#.
 */
Configuration getRuntimeConfiguration() {
  final Configuration runtimeConfiguration = LocalRuntimeConfiguration.CONF
      .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, Integer.toString(maxNumberOfConcurrentEvaluators))
      .set(LocalRuntimeConfiguration.RUNTIME_ROOT_FOLDER, runtimeRootFolder.getAbsolutePath())
      .build();

  final ArrayList<String> driverLaunchCommandPrefixList = new ArrayList<>();
  driverLaunchCommandPrefixList.add(
      new File(driverFolder,
          new REEFFileNames().getDriverLauncherExeFile().toString()
      ).toString());

  final Configuration userProviderConfiguration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindSetEntry(DriverConfigurationProviders.class, TcpPortConfigurationProvider.class)
      .bindSetEntry(DriverConfigurationProviders.class, LoopbackLocalAddressProvider.class)
      .bindNamedParameter(TcpPortRangeBegin.class, Integer.toString(tcpBeginPort))
      .bindNamedParameter(TcpPortRangeCount.class, Integer.toString(tcpRangeCount))
      .bindNamedParameter(TcpPortRangeTryCount.class, Integer.toString(tcpTryCount))
      .bindNamedParameter(JobSubmissionDirectory.class, runtimeRootFolder.getAbsolutePath())
      .bindList(DriverLaunchCommandPrefix.class, driverLaunchCommandPrefixList)
      .build();

  return Configurations.merge(runtimeConfiguration, userProviderConfiguration);
}
 

/**
 * Parse command line arguments and create TANG configuration ready to be submitted to REEF.
 *
 * @param commandLineConf Parsed command line arguments, as passed into main().
 * @return (immutable) TANG Configuration object.
 * @throws BindException      if configuration commandLineInjector fails.
 * @throws InjectionException if configuration commandLineInjector fails.
 */
private static Configuration getClientConfiguration(
    final Configuration commandLineConf, final boolean isLocal) throws BindException, InjectionException {

  final Configuration runtimeConfiguration;

  if (isLocal) {
    LOG.log(Level.FINE, "Running on the local runtime");
    runtimeConfiguration = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
        .build();
  } else {
    LOG.log(Level.FINE, "Running on YARN");
    runtimeConfiguration = YarnClientConfiguration.CONF.build();
  }

  return Configurations.merge(runtimeConfiguration, cloneCommandLineConfiguration(commandLineConf));
}
 

/**
 * Get deploy mode configuration.
 *
 * @param jobConf job configuration to get deploy mode.
 * @return deploy mode configuration.
 * @throws InjectionException exception while injection.
 */
private static Configuration getDeployModeConf(final Configuration jobConf) throws InjectionException {
  final Injector injector = TANG.newInjector(jobConf);
  final String deployMode = injector.getNamedInstance(JobConf.DeployMode.class);
  switch (deployMode) {
    case "local":
      return LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, LOCAL_NUMBER_OF_EVALUATORS)
        .build();
    case "yarn":
      return YarnClientConfiguration.CONF
        .set(YarnClientConfiguration.JVM_HEAP_SLACK, injector.getNamedInstance(JobConf.JVMHeapSlack.class)
          + injector.getNamedInstance(JobConf.MaxOffheapRatio.class))
        // Off-heap memory size is added to memory slack so that JVM heap region does not invade the off-heap region.
        .build();
    default:
      throw new UnsupportedOperationException(deployMode);
  }
}
 

@Override
public Configuration getRuntimeConfiguration(
    final ClientProtocol.DriverClientConfiguration driverClientConfiguration) {
  ConfigurationModule localRuntimeCM = LocalRuntimeConfiguration.CONF;
  if (driverClientConfiguration.getLocalRuntime().getMaxNumberOfEvaluators() > 0) {
    localRuntimeCM = localRuntimeCM.set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS,
        driverClientConfiguration.getLocalRuntime().getMaxNumberOfEvaluators());
  }
  if (StringUtils.isNotEmpty(driverClientConfiguration.getLocalRuntime().getRuntimeRootFolder())) {
    localRuntimeCM = localRuntimeCM.set(LocalRuntimeConfiguration.RUNTIME_ROOT_FOLDER,
        driverClientConfiguration.getLocalRuntime().getRuntimeRootFolder());
  }
  if (driverClientConfiguration.getLocalRuntime().getJvmHeapSlack() > 0.0) {
    localRuntimeCM = localRuntimeCM.set(LocalRuntimeConfiguration.JVM_HEAP_SLACK,
        driverClientConfiguration.getLocalRuntime().getJvmHeapSlack());
  }
  if (StringUtils.isNotEmpty(driverClientConfiguration.getDriverJobSubmissionDirectory())) {
    localRuntimeCM = localRuntimeCM.set(LocalRuntimeConfiguration.RUNTIME_ROOT_FOLDER,
        driverClientConfiguration.getDriverJobSubmissionDirectory());
  }
  return localRuntimeCM.build();
}
 

/**
 * Parse command line arguments and create TANG configuration ready to be submitted to REEF.
 *
 * @param args Command line arguments, as passed into main().
 * @return (immutable) TANG Configuration object.
 * @throws org.apache.reef.tang.exceptions.BindException      if configuration commandLineInjector fails.
 * @throws org.apache.reef.tang.exceptions.InjectionException if configuration commandLineInjector fails.
 * @throws java.io.IOException                                error reading the configuration.
 */
private static Configuration getClientConfiguration(final String[] args)
    throws BindException, InjectionException, IOException {

  try (LoggingScope ls = LoggingScopeFactory.getNewLoggingScope(Level.INFO, "Launch::getClientConfiguration")) {
    final Configuration commandLineConf = parseCommandLine(args);

    final Configuration clientConfiguration = ClientConfiguration.CONF
        .set(ClientConfiguration.ON_JOB_COMPLETED, JobClient.CompletedJobHandler.class)
        .set(ClientConfiguration.ON_JOB_FAILED, JobClient.FailedJobHandler.class)
        .set(ClientConfiguration.ON_RUNTIME_ERROR, JobClient.RuntimeErrorHandler.class)
        .build();

    final Injector commandLineInjector = Tang.Factory.getTang().newInjector(commandLineConf);
    final boolean isLocal = commandLineInjector.getNamedInstance(Local.class);
    final Configuration runtimeConfiguration;
    if (isLocal) {
      LOG.log(Level.INFO, "Running on the local runtime");
      runtimeConfiguration = LocalRuntimeConfiguration.CONF
          .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
          .build();
    } else {
      LOG.log(Level.INFO, "Running on YARN");
      runtimeConfiguration = YarnClientConfiguration.CONF.build();
    }

    return Tang.Factory.getTang()
        .newConfigurationBuilder(runtimeConfiguration, clientConfiguration,
            cloneCommandLineConfiguration(commandLineConf))
        .build();
  }
}
 

/**
 * Start Hello REEF job. Runs method runHelloCLR().
 *
 * @param args command line parameters.
 * @throws org.apache.reef.tang.exceptions.BindException      configuration error.
 * @throws org.apache.reef.tang.exceptions.InjectionException configuration error.
 */
public static void main(final String[] args) throws BindException, InjectionException {
  final Configuration runtimeConfiguration = LocalRuntimeConfiguration.CONF
      .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, 2)
      .build();

  final File dotNetFolder = new File(args[0]).getAbsoluteFile();
  final LauncherStatus status = runHelloCLR(runtimeConfiguration, JOB_TIMEOUT, dotNetFolder);
  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

/**
 * Launch a Vortex job using the local runtime.
 */
public static LauncherStatus launchLocal(final VortexJobConf vortexConf) {
  final Configuration runtimeConf = LocalRuntimeConfiguration.CONF
      .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
      .build();
  return launch(runtimeConf, vortexConf.getConfiguration());
}
 

@Test
public void testHttpServer() throws BindException, InjectionException {

  final Configuration runtimeConfiguration = LocalRuntimeConfiguration.CONF
      .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, 2)
      .set(LocalRuntimeConfiguration.RUNTIME_ROOT_FOLDER, "target/REEF_LOCAL_RUNTIME")
      .build();

  final LauncherStatus status = HelloREEFHttp.runHelloReef(runtimeConfiguration, 10 * 1000);
  Assert.assertEquals(LauncherStatus.FORCE_CLOSED, status); // must be force closed by timeout
}
 

/**
 * Main program.
 *
 * @param args
 * @throws InjectionException
 */
public static void main(final String[] args) throws InjectionException {
  final Configuration runtimeConfiguration = LocalRuntimeConfiguration.CONF
      .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
      .build();
  runHelloReef(runtimeConfiguration, HelloREEFHttp.JOB_TIMEOUT);
}
 

/**
 * Parse command line arguments and create TANG configuration ready to be submitted to REEF.
 *
 * @param args Command line arguments, as passed into main().
 * @return (immutable) TANG Configuration object.
 * @throws BindException      if configuration commandLineInjector fails.
 * @throws InjectionException if configuration commandLineInjector fails.
 * @throws IOException        error reading the configuration.
 */
private static Configuration getClientConfiguration(final String[] args)
    throws BindException, InjectionException, IOException {
  final Configuration commandLineConf = parseCommandLine(args);

  final Configuration clientConfiguration = ClientConfiguration.CONF
      .set(ClientConfiguration.ON_JOB_RUNNING, SuspendClient.RunningJobHandler.class)
      .set(ClientConfiguration.ON_JOB_FAILED, SuspendClient.FailedJobHandler.class)
      .set(ClientConfiguration.ON_JOB_COMPLETED, SuspendClient.CompletedJobHandler.class)
      .set(ClientConfiguration.ON_RUNTIME_ERROR, SuspendClient.RuntimeErrorHandler.class)
      .build();

  final Injector commandLineInjector = Tang.Factory.getTang().newInjector(commandLineConf);
  final boolean isLocal = commandLineInjector.getNamedInstance(Local.class);
  final Configuration runtimeConfiguration;
  if (isLocal) {
    LOG.log(Level.INFO, "Running on the local runtime");
    runtimeConfiguration = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
        .build();
  } else {
    LOG.log(Level.INFO, "Running on YARN");
    runtimeConfiguration = YarnClientConfiguration.CONF.build();
  }

  return Configurations.merge(
      runtimeConfiguration, clientConfiguration, cloneCommandLineConfiguration(commandLineConf));
}
 

/**
 * Main program.
 * @param args
 * @throws InjectionException
 */
public static void main(final String[] args) throws InjectionException, IOException, ParseException {
  final Configuration runtimeConfiguration = LocalRuntimeConfiguration.CONF
      .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
      .build();
  runTaskScheduler(runtimeConfiguration, args);
}
 

/**
 * @return (immutable) TANG Configuration object.
 */
private static Configuration getRunTimeConfiguration() {
  final Configuration runtimeConfiguration;
  if (local) {
    LOG.log(Level.INFO, "Running Broadcast example using group API on the local runtime");
    runtimeConfiguration = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
        .build();
  } else {
    LOG.log(Level.INFO, "Running Broadcast example using group API on YARN");
    runtimeConfiguration = YarnClientConfiguration.CONF.build();
  }
  return runtimeConfiguration;
}
 

/**
 * @param isLocal true for local runtime, or false for YARN runtime.
 * @return The runtime configuration
 */
private static Configuration getRuntimeConf(final boolean isLocal) {
  final Configuration runtimeConf;
  if (isLocal) {
    LOG.log(Level.INFO, "Running the output service demo on the local runtime");
    runtimeConf = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, 3)
        .build();
  } else {
    LOG.log(Level.INFO, "Running the output service demo on YARN");
    runtimeConf = YarnClientConfiguration.CONF.build();
  }
  return runtimeConf;
}
 

/**
 * @return the configuration of the runtime
 */
private static Configuration getRuntimeConfiguration() {
  return LocalRuntimeConfiguration.CONF
      .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
      .build();
}
 

/**
 * @return the configuration of the runtime
 */
private static Configuration getRuntimeConfiguration() {
  return LocalRuntimeConfiguration.CONF
      .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
      .build();
}
 

public static void main(final String[] args)
    throws InjectionException, BindException, IOException {

  final Tang tang = Tang.Factory.getTang();

  final JavaConfigurationBuilder cb = tang.newConfigurationBuilder();

  new CommandLine(cb)
      .registerShortNameOfClass(Local.class)
      .registerShortNameOfClass(TimeOut.class)
      .registerShortNameOfClass(DataLoadingREEF.InputDir.class)
      .processCommandLine(args);

  final Injector injector = tang.newInjector(cb.build());

  final boolean isLocal = injector.getNamedInstance(Local.class);
  final int jobTimeout = injector.getNamedInstance(TimeOut.class) * 60 * 1000;
  final String inputDir = injector.getNamedInstance(DataLoadingREEF.InputDir.class);

  final Configuration runtimeConfiguration;
  if (isLocal) {
    LOG.log(Level.INFO, "Running Data Loading demo on the local runtime");
    runtimeConfiguration = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
        .build();
  } else {
    LOG.log(Level.INFO, "Running Data Loading demo on YARN");
    runtimeConfiguration = YarnClientConfiguration.CONF.build();
  }

  final EvaluatorRequest computeRequest = EvaluatorRequest.newBuilder()
      .setNumber(NUM_COMPUTE_EVALUATORS)
      .setMemory(512)
      .setNumberOfCores(1)
      .build();

  final EvaluatorRequest dataRequest = EvaluatorRequest.newBuilder()
      .setMemory(512)
      .setNumberOfCores(1)
      .build();

  final Configuration dataLoadConfiguration = new DataLoadingRequestBuilder()
      .setInputFormatClass(TextInputFormat.class)
      .setInputPath(inputDir)
      .setNumberOfDesiredSplits(NUM_SPLITS)
      .addComputeRequest(computeRequest)
      .addDataRequest(dataRequest)
      .setDriverConfigurationModule(DriverConfiguration.CONF
          .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(LineCounter.class))
          .set(DriverConfiguration.ON_CONTEXT_ACTIVE, LineCounter.ContextActiveHandler.class)
          .set(DriverConfiguration.ON_TASK_COMPLETED, LineCounter.TaskCompletedHandler.class)
          .set(DriverConfiguration.DRIVER_IDENTIFIER, "DataLoadingREEF"))
      .build();

  final LauncherStatus state =
      DriverLauncher.getLauncher(runtimeConfiguration).run(dataLoadConfiguration, jobTimeout);

  LOG.log(Level.INFO, "REEF job completed: {0}", state);
}
 

/**
 * Start the distributed shell job.
 * @param args command line parameters.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws InjectionException, IOException {

  final JavaConfigurationBuilder driverConfigBuilder = TANG.newConfigurationBuilder(STATIC_DRIVER_CONFIG);

  new CommandLine(driverConfigBuilder)
      .registerShortNameOfClass(Command.class)
      .registerShortNameOfClass(NumEvaluators.class)
      .registerShortNameOfClass(RuntimeName.class)
      .processCommandLine(args);

  final Configuration driverConfig = driverConfigBuilder.build();

  final Injector injector = TANG.newInjector(driverConfig);

  final int numEvaluators = injector.getNamedInstance(NumEvaluators.class);
  final String runtimeName = injector.getNamedInstance(RuntimeName.class);
  final String command = injector.getNamedInstance(Command.class);

  LOG.log(Level.INFO, "REEF distributed shell: {0} evaluators on {1} runtime :: {2}",
      new Object[] {numEvaluators, runtimeName, command});

  final Configuration runtimeConfig;

  switch (runtimeName) {
  case "local":
    runtimeConfig = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, numEvaluators)
        .build();
    break;
  case "yarn":
    runtimeConfig = YarnClientConfiguration.CONF.build();
    break;
  default:
    LOG.log(Level.SEVERE, "Unknown runtime: {0}", runtimeName);
    throw new IllegalArgumentException("Unknown runtime: " + runtimeName);
  }

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfig).run(driverConfig, JOB_TIMEOUT);

  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

public static void main(final String[] args) throws Exception {

    final BGDClient bgdClient = BGDClient.fromCommandLine(args);

    final Configuration runtimeConfiguration = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, "" + MAX_NUMBER_OF_EVALUATORS)
        .build();

    final String jobName = System.getProperty("user.name") + "-" + "ResourceAwareBGDLocal";

    final LauncherStatus status = bgdClient.run(runtimeConfiguration, jobName, TIMEOUT);

    LOG.log(Level.INFO, "OUT: Status = {0}", status);
  }