org.apache.reef.tang.formats.CommandLine Java Examples

/**
 * Get job configuration.
 * @param args arguments to be processed as command line.
 * @return job configuration.
 * @throws IOException exception while processing command line.
 * @throws InjectionException exception while injection.
 */
public static Configuration getJobConf(final String[] args) throws IOException, InjectionException {
  final JavaConfigurationBuilder confBuilder = TANG.newConfigurationBuilder();
  final CommandLine cl = new CommandLine(confBuilder);
  cl.registerShortNameOfClass(JobConf.JobId.class);
  cl.registerShortNameOfClass(JobConf.UserMainClass.class);
  cl.registerShortNameOfClass(JobConf.UserMainArguments.class);
  cl.registerShortNameOfClass(JobConf.DAGDirectory.class);
  cl.registerShortNameOfClass(JobConf.OptimizationPolicy.class);
  cl.registerShortNameOfClass(JobConf.DeployMode.class);
  cl.registerShortNameOfClass(JobConf.DriverMemMb.class);
  cl.registerShortNameOfClass(JobConf.ExecutorJsonPath.class);
  cl.registerShortNameOfClass(JobConf.JVMHeapSlack.class);
  cl.registerShortNameOfClass(JobConf.IORequestHandleThreadsTotal.class);
  cl.registerShortNameOfClass(JobConf.SchedulerTimeoutMs.class);
  cl.registerShortNameOfClass(JobConf.MaxScheduleAttempt.class);
  cl.registerShortNameOfClass(JobConf.FileDirectory.class);
  cl.registerShortNameOfClass(JobConf.GlusterVolumeDirectory.class);
  cl.registerShortNameOfClass(JobConf.PartitionTransportServerPort.class);
  cl.registerShortNameOfClass(JobConf.PartitionTransportServerBacklog.class);
  cl.registerShortNameOfClass(JobConf.PartitionTransportServerNumListeningThreads.class);
  cl.registerShortNameOfClass(JobConf.PartitionTransportServerNumWorkingThreads.class);
  cl.registerShortNameOfClass(JobConf.PartitionTransportClientNumThreads.class);
  cl.processCommandLine(args);
  return confBuilder.build();
}
 

public static void main(final String[] args) throws Exception {
  final Tang tang = Tang.Factory.getTang();
  final JavaConfigurationBuilder cb = tang.newConfigurationBuilder();
  final CommandLine cl = new CommandLine(cb);
  cl.registerShortNameOfClass(Timer.Seconds.class);
  cl.processCommandLine(args);
  final Configuration conf = cb.build();
  System.out.println("start conf");
  final AvroConfigurationSerializer avroSerializer = new AvroConfigurationSerializer();
  System.out.println(avroSerializer.toString(conf));
  System.out.println("end conf");
  final InjectorImpl injector = (InjectorImpl) tang.newInjector(conf);
  final InjectionPlan<Timer> ip = injector.getInjectionPlan(Timer.class);
  System.out.println(ip.toPrettyString());
  System.out.println("Number of plans:" + ip.getNumAlternatives());
  final Timer timer = injector.getInstance(Timer.class);
  System.out.println("Tick...");
  timer.sleep();
  System.out.println("Tock.");
}
 

public static void main(final String[] args)
    throws InjectionException, BindException, IOException {

  final Tang tang = Tang.Factory.getTang();
  final JavaConfigurationBuilder cb = tang.newConfigurationBuilder();
  new CommandLine(cb)
      .registerShortNameOfClass(Local.class)
      .registerShortNameOfClass(TimeOut.class)
      .registerShortNameOfClass(OutputDir.class)
      .processCommandLine(args);

  final Injector injector = tang.newInjector(cb.build());
  final boolean isLocal = injector.getNamedInstance(Local.class);
  final String outputDir = injector.getNamedInstance(OutputDir.class);
  final int jobTimeout = injector.getNamedInstance(TimeOut.class) * 60 * 1000;

  final Configuration driverConf = getDriverConf();
  final Configuration outputServiceConf = getOutputServiceConf(isLocal, outputDir);
  final Configuration submittedConfiguration = Tang.Factory.getTang()
      .newConfigurationBuilder(driverConf, outputServiceConf)
      .build();
  final LauncherStatus state = DriverLauncher.getLauncher(getRuntimeConf(isLocal))
      .run(submittedConfiguration, jobTimeout);

  LOG.log(Level.INFO, "REEF job completed: {0}", state);
}
 

public static void main(final String[] argv) throws IOException, BindException, InjectionException {
  @SuppressWarnings("unchecked") final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder(
      (Class<? extends ExternalConstructor<?>>[]) new Class[]{FileParser.class});
  final CommandLine cl = new CommandLine(cb);
  cl.processCommandLine(argv,
      Target.class,
      ClassHierarchyIn.class,
      ConfigurationIn.class,
      ConfigurationOut.class);
  final ValidateConfiguration bip =
      Tang.Factory.getTang().newInjector(cb.build()).getInstance(ValidateConfiguration.class);
  bip.validatePlan();
}
 

/**
 * Get job configuration.
 *
 * @param args arguments to be processed as command line.
 * @return job configuration.
 * @throws IOException        exception while processing command line.
 */
@VisibleForTesting
public static Configuration getJobConf(final String[] args) throws IOException {
  final JavaConfigurationBuilder confBuilder = TANG.newConfigurationBuilder();
  final CommandLine cl = new CommandLine(confBuilder);
  cl.registerShortNameOfClass(JobConf.JobId.class);
  cl.registerShortNameOfClass(JobConf.UserMainClass.class);
  cl.registerShortNameOfClass(JobConf.UserMainArguments.class);
  cl.registerShortNameOfClass(JobConf.DAGDirectory.class);
  cl.registerShortNameOfClass(JobConf.OptimizationPolicy.class);
  cl.registerShortNameOfClass(JobConf.DeployMode.class);
  cl.registerShortNameOfClass(JobConf.ExecutorType.class);
  cl.registerShortNameOfClass(JobConf.DriverMemMb.class);
  cl.registerShortNameOfClass(JobConf.ExecutorJSONPath.class);
  cl.registerShortNameOfClass(JobConf.BandwidthJSONPath.class);
  cl.registerShortNameOfClass(JobConf.JVMHeapSlack.class);
  cl.registerShortNameOfClass(JobConf.IORequestHandleThreadsTotal.class);
  cl.registerShortNameOfClass(JobConf.MaxTaskAttempt.class);
  cl.registerShortNameOfClass(JobConf.FileDirectory.class);
  cl.registerShortNameOfClass(JobConf.GlusterVolumeDirectory.class);
  cl.registerShortNameOfClass(JobConf.PartitionTransportServerPort.class);
  cl.registerShortNameOfClass(JobConf.PartitionTransportServerBacklog.class);
  cl.registerShortNameOfClass(JobConf.PartitionTransportServerNumListeningThreads.class);
  cl.registerShortNameOfClass(JobConf.PartitionTransportServerNumWorkingThreads.class);
  cl.registerShortNameOfClass(JobConf.PartitionTransportClientNumThreads.class);
  cl.registerShortNameOfClass(JobConf.MaxNumDownloadsForARuntimeEdge.class);
  cl.registerShortNameOfClass(JobConf.SchedulerImplClassName.class);
  cl.registerShortNameOfClass(JobConf.ScheduleSerThread.class);
  cl.registerShortNameOfClass(JobConf.MaxOffheapRatio.class);
  cl.registerShortNameOfClass(JobConf.ChunkSizeKb.class);
  cl.processCommandLine(args);
  return confBuilder.build();
}
 

/**
 * @param args command line arguments.
 * @throws BindException      configuration error.
 * @throws InjectionException configuration error.
 * @throws IOException        cannot process command line parameters.
 */
public static void main(final String[] args)
    throws BindException, InjectionException, IOException {

  final Tang tang = Tang.Factory.getTang();
  final ConfigurationBuilder confBuilder = tang.newConfigurationBuilder();

  new CommandLine(confBuilder).processCommandLine(args);
  final Configuration config = confBuilder.build();

  final Injector injector = tang.newInjector(config);
  final PrintTypeHierarchy myself = injector.getInstance(PrintTypeHierarchy.class);

  try (Writer out = new OutputStreamWriter(
          new FileOutputStream("type-hierarchy.dot"), StandardCharsets.UTF_8)) {
    out.write(GraphvizConfigVisitor.getGraphvizString(config, true, true));
  }

  final InjectionPlan<PrintTypeHierarchy> plan =
      injector.getInjectionPlan(PrintTypeHierarchy.class);

  try (Writer out = new OutputStreamWriter(
          new FileOutputStream("injection-plan.dot"), StandardCharsets.UTF_8)) {
    out.write(GraphvizInjectionPlanVisitor.getGraphvizString(plan, true));
  }

  System.out.println(myself);
}
 

/**
 * Parse the command line arguments.
 *
 * @param args command line arguments, as passed to main()
 * @return Configuration object.
 * @throws org.apache.reef.tang.exceptions.BindException configuration error.
 * @throws java.io.IOException                           error reading the configuration.
 */
private static Configuration parseCommandLine(final String[] args)
    throws BindException, IOException {
  final JavaConfigurationBuilder confBuilder = Tang.Factory.getTang().newConfigurationBuilder();
  final CommandLine cl = new CommandLine(confBuilder);
  cl.registerShortNameOfClass(Local.class);
  cl.registerShortNameOfClass(NumRuns.class);
  cl.registerShortNameOfClass(WaitTimeForDriver.class);
  cl.registerShortNameOfClass(DriverMemoryInMb.class);
  cl.registerShortNameOfClass(DriverIdentifier.class);
  cl.registerShortNameOfClass(DriverJobSubmissionDirectory.class);
  cl.registerShortNameOfClass(Submit.class);
  cl.processCommandLine(args);
  return confBuilder.build();
}
 

public static org.apache.reef.tang.Configuration parseCommandLine(final String[] args) throws IOException {
  final JavaConfigurationBuilder confBuilder = Tang.Factory.getTang().newConfigurationBuilder();

  new CommandLine(confBuilder)
      .registerShortNameOfClass(MesosExecutorId.class)
      .processCommandLine(args);

  return confBuilder.build();
}
 

/**
 * @param args command line arguments, as passed to main()
 * @return Configuration object.
 */
private static Configuration parseCommandLine(final String[] args)
    throws IOException, BindException {
  final JavaConfigurationBuilder confBuilder = Tang.Factory.getTang().newConfigurationBuilder();
  final CommandLine cl = new CommandLine(confBuilder);
  cl.registerShortNameOfClass(Local.class);
  cl.registerShortNameOfClass(NumCycles.class);
  cl.registerShortNameOfClass(Delay.class);
  cl.registerShortNameOfClass(SuspendClientControl.Port.class);
  cl.processCommandLine(args);
  return confBuilder.build();
}
 

/**
 * Start Hello REEF job.
 *
 * @param args command line parameters.
 * @throws BindException      configuration error.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws BindException, InjectionException {

  final Tang tang = Tang.Factory.getTang();

  final JavaConfigurationBuilder cb = tang.newConfigurationBuilder();

  try{
    new CommandLine(cb)
        .registerShortNameOfClass(NodeListFilePath.class)
        .registerShortNameOfClass(SshPortNum.class)
        .processCommandLine(args);
  } catch(final IOException ex) {
    LOG.log(Level.SEVERE, "Missing parameter 'nodelist' or wrong parameter input.");
    throw new RuntimeException("Missing parameter 'nodelist' or wrong parameter input: ", ex);
  }

  final Injector injector = tang.newInjector(cb.build());

  final String nodeListFilePath = injector.getNamedInstance(NodeListFilePath.class);
  final int sshPortNum = injector.getNamedInstance(SshPortNum.class);

  final Configuration runtimeConf = getRuntimeConfiguration(nodeListFilePath, sshPortNum);
  final Configuration driverConf = getDriverConfiguration();

  final LauncherStatus status = DriverLauncher
      .getLauncher(runtimeConf)
      .run(driverConf, JOB_TIMEOUT);
  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

/**
 * Run the Task scheduler. If '-retain true' option is passed via command line,
 * the scheduler reuses evaluators to submit new Tasks.
 * @param runtimeConf The runtime configuration (e.g. Local, YARN, etc)
 * @param args Command line arguments.
 * @throws InjectionException
 * @throws java.io.IOException
 */
public static void runTaskScheduler(final Configuration runtimeConf, final String[] args)
    throws InjectionException, IOException, ParseException {
  final Tang tang = Tang.Factory.getTang();

  final Configuration commandLineConf = CommandLine.parseToConfiguration(args, Retain.class);

  // Merge the configurations to run Driver
  final Configuration driverConf = Configurations.merge(getDriverConf(), getHttpConf(), commandLineConf);

  final REEF reef = tang.newInjector(runtimeConf).getInstance(REEF.class);
  reef.submit(driverConf);
}
 

public static final BGDClient fromCommandLine(final String[] args) throws Exception {
  final JavaConfigurationBuilder configurationBuilder = Tang.Factory.getTang().newConfigurationBuilder();
  final CommandLine commandLine = new CommandLine(configurationBuilder)
      .registerShortNameOfClass(InputDir.class)
      .registerShortNameOfClass(Timeout.class)
      .registerShortNameOfClass(EvaluatorMemory.class)
      .registerShortNameOfClass(NumSplits.class)
      .registerShortNameOfClass(TreeTopologyFanOut.class);
  BGDControlParameters.registerShortNames(commandLine);
  commandLine.processCommandLine(args);
  return Tang.Factory.getTang().newInjector(configurationBuilder.build()).getInstance(BGDClient.class);
}
 

public static CommandLine registerShortNames(final CommandLine commandLine) {
  return commandLine
      .registerShortNameOfClass(ModelDimensions.class)
      .registerShortNameOfClass(Lambda.class)
      .registerShortNameOfClass(Eps.class)
      .registerShortNameOfClass(Eta.class)
      .registerShortNameOfClass(ProbabilityOfSuccessfulIteration.class)
      .registerShortNameOfClass(Iterations.class)
      .registerShortNameOfClass(EnableRampup.class)
      .registerShortNameOfClass(MinParts.class)
      .registerShortNameOfClass(LossFunctionType.class);
}
 

private static Configuration parseCommandLine(final String[] aArgs) {
  final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder();
  try {
    final CommandLine cl = new CommandLine(cb);
    cl.registerShortNameOfClass(Local.class);
    cl.registerShortNameOfClass(ModelDimensions.class);
    cl.registerShortNameOfClass(NumberOfReceivers.class);
    cl.processCommandLine(aArgs);
  } catch (final IOException ex) {
    final String msg = "Unable to parse command line";
    LOG.log(Level.SEVERE, msg, ex);
    throw new RuntimeException(msg, ex);
  }
  return cb.build();
}
 

/**
 * Parse the command line arguments.
 *
 * @param args command line arguments, as passed to main()
 * @return Configuration object.
 * @throws BindException configuration error.
 * @throws IOException   error reading the configuration.
 */
private static Configuration parseCommandLine(final String[] args)
    throws BindException, IOException {
  final JavaConfigurationBuilder confBuilder = Tang.Factory.getTang().newConfigurationBuilder();
  final CommandLine cl = new CommandLine(confBuilder);
  cl.registerShortNameOfClass(Local.class);
  cl.registerShortNameOfClass(Piggyback.class);
  cl.registerShortNameOfClass(NumEvaluators.class);
  cl.registerShortNameOfClass(NumTasks.class);
  cl.registerShortNameOfClass(Delay.class);
  cl.registerShortNameOfClass(JobId.class);
  cl.processCommandLine(args);
  return confBuilder.build();
}
 

private static Configuration parseCommandLine(final String[] aArgs) {
  final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder();
  try {
    new CommandLine(cb)
        .registerShortNameOfClass(Local.class)
        .registerShortNameOfClass(TimeOut.class)
        .processCommandLine(aArgs);
    return cb.build();
  } catch (final BindException | IOException ex) {
    final String msg = "Unable to parse command line";
    LOG.log(Level.SEVERE, msg, ex);
    throw new RuntimeException(msg, ex);
  }
}
 

private static Configuration getConfig(final String[] args) throws IOException, BindException {
  final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder();
  new CommandLine(cb).processCommandLine(args, SuspendClientControl.Port.class, TaskId.class, Command.class);
  return cb.build();
}
 

public static void main(final String[] args)
    throws InjectionException, BindException, IOException {

  final Tang tang = Tang.Factory.getTang();

  final JavaConfigurationBuilder cb = tang.newConfigurationBuilder();

  new CommandLine(cb)
      .registerShortNameOfClass(Local.class)
      .registerShortNameOfClass(TimeOut.class)
      .registerShortNameOfClass(DataLoadingREEF.InputDir.class)
      .processCommandLine(args);

  final Injector injector = tang.newInjector(cb.build());

  final boolean isLocal = injector.getNamedInstance(Local.class);
  final int jobTimeout = injector.getNamedInstance(TimeOut.class) * 60 * 1000;
  final String inputDir = injector.getNamedInstance(DataLoadingREEF.InputDir.class);

  final Configuration runtimeConfiguration;
  if (isLocal) {
    LOG.log(Level.INFO, "Running Data Loading demo on the local runtime");
    runtimeConfiguration = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
        .build();
  } else {
    LOG.log(Level.INFO, "Running Data Loading demo on YARN");
    runtimeConfiguration = YarnClientConfiguration.CONF.build();
  }

  final EvaluatorRequest computeRequest = EvaluatorRequest.newBuilder()
      .setNumber(NUM_COMPUTE_EVALUATORS)
      .setMemory(512)
      .setNumberOfCores(1)
      .build();

  final EvaluatorRequest dataRequest = EvaluatorRequest.newBuilder()
      .setMemory(512)
      .setNumberOfCores(1)
      .build();

  final Configuration dataLoadConfiguration = new DataLoadingRequestBuilder()
      .setInputFormatClass(TextInputFormat.class)
      .setInputPath(inputDir)
      .setNumberOfDesiredSplits(NUM_SPLITS)
      .addComputeRequest(computeRequest)
      .addDataRequest(dataRequest)
      .setDriverConfigurationModule(DriverConfiguration.CONF
          .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(LineCounter.class))
          .set(DriverConfiguration.ON_CONTEXT_ACTIVE, LineCounter.ContextActiveHandler.class)
          .set(DriverConfiguration.ON_TASK_COMPLETED, LineCounter.TaskCompletedHandler.class)
          .set(DriverConfiguration.DRIVER_IDENTIFIER, "DataLoadingREEF"))
      .build();

  final LauncherStatus state =
      DriverLauncher.getLauncher(runtimeConfiguration).run(dataLoadConfiguration, jobTimeout);

  LOG.log(Level.INFO, "REEF job completed: {0}", state);
}
 

/**
 * Start the distributed shell job.
 * @param args command line parameters.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws InjectionException, IOException {

  final JavaConfigurationBuilder driverConfigBuilder = TANG.newConfigurationBuilder(STATIC_DRIVER_CONFIG);

  new CommandLine(driverConfigBuilder)
      .registerShortNameOfClass(Command.class)
      .registerShortNameOfClass(NumEvaluators.class)
      .registerShortNameOfClass(RuntimeName.class)
      .processCommandLine(args);

  final Configuration driverConfig = driverConfigBuilder.build();

  final Injector injector = TANG.newInjector(driverConfig);

  final int numEvaluators = injector.getNamedInstance(NumEvaluators.class);
  final String runtimeName = injector.getNamedInstance(RuntimeName.class);
  final String command = injector.getNamedInstance(Command.class);

  LOG.log(Level.INFO, "REEF distributed shell: {0} evaluators on {1} runtime :: {2}",
      new Object[] {numEvaluators, runtimeName, command});

  final Configuration runtimeConfig;

  switch (runtimeName) {
  case "local":
    runtimeConfig = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, numEvaluators)
        .build();
    break;
  case "yarn":
    runtimeConfig = YarnClientConfiguration.CONF.build();
    break;
  default:
    LOG.log(Level.SEVERE, "Unknown runtime: {0}", runtimeName);
    throw new IllegalArgumentException("Unknown runtime: " + runtimeName);
  }

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfig).run(driverConfig, JOB_TIMEOUT);

  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}