org.apache.reef.runtime.yarn.client.YarnClientConfiguration Java Examples

/**
 * Get deploy mode configuration.
 * @param jobConf job configuration to get deploy mode.
 * @return deploy mode configuration.
 * @throws InjectionException exception while injection.
 */
public static Configuration getDeployModeConf(final Configuration jobConf) throws InjectionException {
  final Injector injector = TANG.newInjector(jobConf);
  final String deployMode = injector.getNamedInstance(JobConf.DeployMode.class);
  switch (deployMode) {
    case "local":
      return LocalRuntimeConfiguration.CONF
          .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, LOCAL_NUMBER_OF_EVALUATORS)
          .build();
    case "yarn":
      return YarnClientConfiguration.CONF
          .set(YarnClientConfiguration.JVM_HEAP_SLACK, injector.getNamedInstance(JobConf.JVMHeapSlack.class))
          .build();
    default:
      throw new UnsupportedOperationException(deployMode);
  }
}
 

/**
 * Get deploy mode configuration.
 *
 * @param jobConf job configuration to get deploy mode.
 * @return deploy mode configuration.
 * @throws InjectionException exception while injection.
 */
private static Configuration getDeployModeConf(final Configuration jobConf) throws InjectionException {
  final Injector injector = TANG.newInjector(jobConf);
  final String deployMode = injector.getNamedInstance(JobConf.DeployMode.class);
  switch (deployMode) {
    case "local":
      return LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, LOCAL_NUMBER_OF_EVALUATORS)
        .build();
    case "yarn":
      return YarnClientConfiguration.CONF
        .set(YarnClientConfiguration.JVM_HEAP_SLACK, injector.getNamedInstance(JobConf.JVMHeapSlack.class)
          + injector.getNamedInstance(JobConf.MaxOffheapRatio.class))
        // Off-heap memory size is added to memory slack so that JVM heap region does not invade the off-heap region.
        .build();
    default:
      throw new UnsupportedOperationException(deployMode);
  }
}
 

/**
 * @return (immutable) TANG Configuration object.
 * @throws BindException      if configuration injector fails.
 * @throws InjectionException if the Local.class parameter is not injected.
 */
private static Configuration getRunTimeConfiguration(final boolean isLocal) throws BindException {

  final Configuration runtimeConfiguration;

  if (isLocal) {
    LOG.log(Level.INFO, "Running Failure demo on the local runtime");
    runtimeConfiguration = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
        .build();
  } else {
    LOG.log(Level.INFO, "Running Failure demo on YARN");
    runtimeConfiguration = YarnClientConfiguration.CONF.build();
  }

  return runtimeConfiguration;
}
 

/**
 * Parse command line arguments and create TANG configuration ready to be submitted to REEF.
 *
 * @param commandLineConf Parsed command line arguments, as passed into main().
 * @return (immutable) TANG Configuration object.
 * @throws BindException      if configuration commandLineInjector fails.
 * @throws InjectionException if configuration commandLineInjector fails.
 */
private static Configuration getClientConfiguration(
    final Configuration commandLineConf, final boolean isLocal) throws BindException, InjectionException {

  final Configuration runtimeConfiguration;

  if (isLocal) {
    LOG.log(Level.FINE, "Running on the local runtime");
    runtimeConfiguration = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
        .build();
  } else {
    LOG.log(Level.FINE, "Running on YARN");
    runtimeConfiguration = YarnClientConfiguration.CONF.build();
  }

  return Configurations.merge(runtimeConfiguration, cloneCommandLineConfiguration(commandLineConf));
}
 

/**
 * Parse command line arguments and create TANG configuration ready to be submitted to REEF.
 *
 * @param args Command line arguments, as passed into main().
 * @return (immutable) TANG Configuration object.
 * @throws org.apache.reef.tang.exceptions.BindException      if configuration commandLineInjector fails.
 * @throws org.apache.reef.tang.exceptions.InjectionException if configuration commandLineInjector fails.
 * @throws java.io.IOException                                error reading the configuration.
 */
private static Configuration getClientConfiguration(final String[] args)
    throws BindException, InjectionException, IOException {

  try (LoggingScope ls = LoggingScopeFactory.getNewLoggingScope(Level.INFO, "Launch::getClientConfiguration")) {
    final Configuration commandLineConf = parseCommandLine(args);

    final Configuration clientConfiguration = ClientConfiguration.CONF
        .set(ClientConfiguration.ON_JOB_COMPLETED, JobClient.CompletedJobHandler.class)
        .set(ClientConfiguration.ON_JOB_FAILED, JobClient.FailedJobHandler.class)
        .set(ClientConfiguration.ON_RUNTIME_ERROR, JobClient.RuntimeErrorHandler.class)
        .build();

    final Injector commandLineInjector = Tang.Factory.getTang().newInjector(commandLineConf);
    final boolean isLocal = commandLineInjector.getNamedInstance(Local.class);
    final Configuration runtimeConfiguration;
    if (isLocal) {
      LOG.log(Level.INFO, "Running on the local runtime");
      runtimeConfiguration = LocalRuntimeConfiguration.CONF
          .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
          .build();
    } else {
      LOG.log(Level.INFO, "Running on YARN");
      runtimeConfiguration = YarnClientConfiguration.CONF.build();
    }

    return Tang.Factory.getTang()
        .newConfigurationBuilder(runtimeConfiguration, clientConfiguration,
            cloneCommandLineConfiguration(commandLineConf))
        .build();
  }
}
 

public Configuration getRuntimeConfiguration(
    final ClientProtocol.DriverClientConfiguration driverClientConfiguration) {
  Configuration yarnConfiguration = YarnClientConfiguration.CONF
      .set(YarnClientConfiguration.UNMANAGED_DRIVER,
          driverClientConfiguration.getYarnRuntime().getUnmangedDriver())
      .set(YarnClientConfiguration.YARN_PRIORITY, driverClientConfiguration.getYarnRuntime().getPriority())
      .set(YarnClientConfiguration.JVM_HEAP_SLACK, 0.0)
      .build();
  if (StringUtils.isNotEmpty(driverClientConfiguration.getYarnRuntime().getFilesystemUrl())) {
    final JavaConfigurationBuilder providerConfig = Tang.Factory.getTang().newConfigurationBuilder()
        .bindNamedParameter(FileSystemUrl.class, driverClientConfiguration.getYarnRuntime().getFilesystemUrl());
    yarnConfiguration = Configurations.merge(yarnConfiguration, providerConfig.build());
  }
  return yarnConfiguration;
}
 

/**
 * Start Hello REEF job. Runs method runHelloReef().
 *
 * @param args command line parameters.
 * @throws org.apache.reef.tang.exceptions.BindException      configuration error.
 * @throws org.apache.reef.tang.exceptions.InjectionException configuration error.
 */
public static void main(final String[] args) throws BindException, InjectionException, IOException {

  final Configuration runtimeConfiguration = YarnClientConfiguration.CONF.build();

  final LauncherStatus status = HelloREEFHttp.runHelloReef(runtimeConfiguration, HelloREEFHttp.JOB_TIMEOUT);
  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

/**
 * Parse command line arguments and create TANG configuration ready to be submitted to REEF.
 *
 * @param args Command line arguments, as passed into main().
 * @return (immutable) TANG Configuration object.
 * @throws BindException      if configuration commandLineInjector fails.
 * @throws InjectionException if configuration commandLineInjector fails.
 * @throws IOException        error reading the configuration.
 */
private static Configuration getClientConfiguration(final String[] args)
    throws BindException, InjectionException, IOException {
  final Configuration commandLineConf = parseCommandLine(args);

  final Configuration clientConfiguration = ClientConfiguration.CONF
      .set(ClientConfiguration.ON_JOB_RUNNING, SuspendClient.RunningJobHandler.class)
      .set(ClientConfiguration.ON_JOB_FAILED, SuspendClient.FailedJobHandler.class)
      .set(ClientConfiguration.ON_JOB_COMPLETED, SuspendClient.CompletedJobHandler.class)
      .set(ClientConfiguration.ON_RUNTIME_ERROR, SuspendClient.RuntimeErrorHandler.class)
      .build();

  final Injector commandLineInjector = Tang.Factory.getTang().newInjector(commandLineConf);
  final boolean isLocal = commandLineInjector.getNamedInstance(Local.class);
  final Configuration runtimeConfiguration;
  if (isLocal) {
    LOG.log(Level.INFO, "Running on the local runtime");
    runtimeConfiguration = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
        .build();
  } else {
    LOG.log(Level.INFO, "Running on YARN");
    runtimeConfiguration = YarnClientConfiguration.CONF.build();
  }

  return Configurations.merge(
      runtimeConfiguration, clientConfiguration, cloneCommandLineConfiguration(commandLineConf));
}
 

/**
 * @param tcpBeginPort  the first tcp port number to try
 * @param tcpRangeCount the number of tcp ports in the range
 * @param tcpTryCount maximum number of tries for port numbers
 * @return the configuration of the runtime
 */
private static Configuration getRuntimeConfiguration(
    final int tcpBeginPort,
    final int tcpRangeCount,
    final int tcpTryCount) {

  return Tang.Factory.getTang().newConfigurationBuilder(YarnClientConfiguration.CONF.build())
      .bindSetEntry(DriverConfigurationProviders.class, TcpPortConfigurationProvider.class)
      .bindNamedParameter(TcpPortRangeBegin.class, Integer.toString(tcpBeginPort))
      .bindNamedParameter(TcpPortRangeCount.class, Integer.toString(tcpRangeCount))
      .bindNamedParameter(TcpPortRangeTryCount.class, Integer.toString(tcpTryCount))
      .build();
}
 

/**
 * Launch the scheduler with YARN client configuration.
 * @param args
 * @throws InjectionException
 * @throws java.io.IOException
 */
public static void main(final String[] args)
    throws InjectionException, IOException, ParseException {
  final Configuration runtimeConfiguration =
      YarnClientConfiguration.CONF.build();
  runTaskScheduler(runtimeConfiguration, args);
}
 

/**
 * @return (immutable) TANG Configuration object.
 */
private static Configuration getRunTimeConfiguration() {
  final Configuration runtimeConfiguration;
  if (local) {
    LOG.log(Level.INFO, "Running Broadcast example using group API on the local runtime");
    runtimeConfiguration = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
        .build();
  } else {
    LOG.log(Level.INFO, "Running Broadcast example using group API on YARN");
    runtimeConfiguration = YarnClientConfiguration.CONF.build();
  }
  return runtimeConfiguration;
}
 

/**
 * @param isLocal true for local runtime, or false for YARN runtime.
 * @return The runtime configuration
 */
private static Configuration getRuntimeConf(final boolean isLocal) {
  final Configuration runtimeConf;
  if (isLocal) {
    LOG.log(Level.INFO, "Running the output service demo on the local runtime");
    runtimeConf = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, 3)
        .build();
  } else {
    LOG.log(Level.INFO, "Running the output service demo on YARN");
    runtimeConf = YarnClientConfiguration.CONF.build();
  }
  return runtimeConf;
}
 

/**
 * @return the configuration of the runtime
 */
private static Configuration getRuntimeConfiguration() {
  return YarnClientConfiguration.CONF.build();
}
 

public static void main(final String[] args)
    throws InjectionException, BindException, IOException {

  final Tang tang = Tang.Factory.getTang();

  final JavaConfigurationBuilder cb = tang.newConfigurationBuilder();

  new CommandLine(cb)
      .registerShortNameOfClass(Local.class)
      .registerShortNameOfClass(TimeOut.class)
      .registerShortNameOfClass(DataLoadingREEF.InputDir.class)
      .processCommandLine(args);

  final Injector injector = tang.newInjector(cb.build());

  final boolean isLocal = injector.getNamedInstance(Local.class);
  final int jobTimeout = injector.getNamedInstance(TimeOut.class) * 60 * 1000;
  final String inputDir = injector.getNamedInstance(DataLoadingREEF.InputDir.class);

  final Configuration runtimeConfiguration;
  if (isLocal) {
    LOG.log(Level.INFO, "Running Data Loading demo on the local runtime");
    runtimeConfiguration = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
        .build();
  } else {
    LOG.log(Level.INFO, "Running Data Loading demo on YARN");
    runtimeConfiguration = YarnClientConfiguration.CONF.build();
  }

  final EvaluatorRequest computeRequest = EvaluatorRequest.newBuilder()
      .setNumber(NUM_COMPUTE_EVALUATORS)
      .setMemory(512)
      .setNumberOfCores(1)
      .build();

  final EvaluatorRequest dataRequest = EvaluatorRequest.newBuilder()
      .setMemory(512)
      .setNumberOfCores(1)
      .build();

  final Configuration dataLoadConfiguration = new DataLoadingRequestBuilder()
      .setInputFormatClass(TextInputFormat.class)
      .setInputPath(inputDir)
      .setNumberOfDesiredSplits(NUM_SPLITS)
      .addComputeRequest(computeRequest)
      .addDataRequest(dataRequest)
      .setDriverConfigurationModule(DriverConfiguration.CONF
          .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(LineCounter.class))
          .set(DriverConfiguration.ON_CONTEXT_ACTIVE, LineCounter.ContextActiveHandler.class)
          .set(DriverConfiguration.ON_TASK_COMPLETED, LineCounter.TaskCompletedHandler.class)
          .set(DriverConfiguration.DRIVER_IDENTIFIER, "DataLoadingREEF"))
      .build();

  final LauncherStatus state =
      DriverLauncher.getLauncher(runtimeConfiguration).run(dataLoadConfiguration, jobTimeout);

  LOG.log(Level.INFO, "REEF job completed: {0}", state);
}
 

/**
 * Start the distributed shell job.
 * @param args command line parameters.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws InjectionException, IOException {

  final JavaConfigurationBuilder driverConfigBuilder = TANG.newConfigurationBuilder(STATIC_DRIVER_CONFIG);

  new CommandLine(driverConfigBuilder)
      .registerShortNameOfClass(Command.class)
      .registerShortNameOfClass(NumEvaluators.class)
      .registerShortNameOfClass(RuntimeName.class)
      .processCommandLine(args);

  final Configuration driverConfig = driverConfigBuilder.build();

  final Injector injector = TANG.newInjector(driverConfig);

  final int numEvaluators = injector.getNamedInstance(NumEvaluators.class);
  final String runtimeName = injector.getNamedInstance(RuntimeName.class);
  final String command = injector.getNamedInstance(Command.class);

  LOG.log(Level.INFO, "REEF distributed shell: {0} evaluators on {1} runtime :: {2}",
      new Object[] {numEvaluators, runtimeName, command});

  final Configuration runtimeConfig;

  switch (runtimeName) {
  case "local":
    runtimeConfig = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, numEvaluators)
        .build();
    break;
  case "yarn":
    runtimeConfig = YarnClientConfiguration.CONF.build();
    break;
  default:
    LOG.log(Level.SEVERE, "Unknown runtime: {0}", runtimeName);
    throw new IllegalArgumentException("Unknown runtime: " + runtimeName);
  }

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfig).run(driverConfig, JOB_TIMEOUT);

  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

public static void main(final String[] args) throws Exception {

    final BGDClient bgdClient = BGDClient.fromCommandLine(args);

    final Configuration runtimeConfiguration = YarnClientConfiguration.CONF
        .set(YarnClientConfiguration.JVM_HEAP_SLACK, "0.1")
        .build();

    final String jobName = System.getProperty("user.name") + "-" + "BR-ResourceAwareBGD-YARN";

    final LauncherStatus status = bgdClient.run(runtimeConfiguration, jobName, TIMEOUT);

    LOG.log(Level.INFO, "OUT: Status = {0}", status);
  }
 

/**
 * Specifies YARN as the runtime for REEF cluster. Override this method to use a different
 * runtime.
 */
Configuration getRuntimeConf() {
  return YarnClientConfiguration.CONF.build();
}