Java Code Examples for org.apache.reef.client.DriverLauncher#getLauncher()

/**
 * Start Hello REEF job with Unmanaged Driver running locally in the same process.
 * @param args command line parameters. Not used.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws InjectionException {

  LOG.log(Level.FINE, "Launching Unmanaged AM: {0}", JAR_PATH);

  try (DriverLauncher client = DriverLauncher.getLauncher(RUNTIME_CONFIG)) {

    final String appId = client.submit(DRIVER_CONFIG, 10000);
    LOG.log(Level.INFO, "Job submitted: {0}", appId);

    final Configuration yarnAmConfig = UnmanagedAmYarnDriverConfiguration.CONF
        .set(UnmanagedAmYarnDriverConfiguration.JOB_IDENTIFIER, appId)
        .set(UnmanagedAmYarnDriverConfiguration.JOB_SUBMISSION_DIRECTORY, DRIVER_ROOT_PATH)
        .build();

    try (REEFEnvironment reef = REEFEnvironment.fromConfiguration(yarnAmConfig, DRIVER_CONFIG)) {
      reef.run();
      final ReefServiceProtos.JobStatusProto status = reef.getLastStatus();
      LOG.log(Level.INFO, "REEF job {0} completed: state {1}", new Object[] {appId, status.getState()});
    }
  }

  ThreadLogger.logThreads(LOG, Level.FINEST, "Threads running after DriverLauncher.close():");
  System.exit(0); // TODO[REEF-1715]: Should be able to exit cleanly at the end of main()
}
 

/**
 * Start Hello REEF job with Unmanaged Driver running locally in the same process.
 * @throws InjectionException configuration error.
 */
@Test
public void testHelloREEF() throws InjectionException {

  Assume.assumeTrue(
      "This test requires a YARN Resource Manager to connect to",
      Boolean.parseBoolean(System.getenv("REEF_TEST_YARN")));

  LOG.log(Level.FINE, "Launching Unmanaged AM: {0}", JAR_PATH);

  try (DriverLauncher client = DriverLauncher.getLauncher(RUNTIME_CONFIG)) {

    final String appId = client.submit(DRIVER_CONFIG, 10000);
    LOG.log(Level.INFO, "Job submitted: {0}", appId);

    final Configuration yarnAmConfig = UnmanagedAmYarnDriverConfiguration.CONF
        .set(UnmanagedAmYarnDriverConfiguration.JOB_IDENTIFIER, appId)
        .set(UnmanagedAmYarnDriverConfiguration.JOB_SUBMISSION_DIRECTORY, DRIVER_ROOT_PATH)
        .build();

    try (REEFEnvironment reef = REEFEnvironment.fromConfiguration(yarnAmConfig, DRIVER_CONFIG)) {

      reef.run();

      final ReefServiceProtos.JobStatusProto status = reef.getLastStatus();
      final ReefServiceProtos.State jobState = status.getState();
      LOG.log(Level.INFO, "REEF job {0} completed: state {1}", new Object[] {appId, jobState});
      Assert.assertEquals("Job state after execution: " + jobState, jobState, ReefServiceProtos.State.DONE);
    }

    final LauncherStatus clientStatus = client.getStatus();
    LOG.log(Level.INFO, "REEF job {0} completed: client status {1}", new Object[] {appId, clientStatus});
    Assert.assertTrue("Job state after execution: " + clientStatus, clientStatus.isSuccess());
  }

  ThreadLogger.logThreads(LOG, Level.FINEST, "Threads running after DriverLauncher.close():");

  LOG.log(Level.INFO, "Clean exit!");
}
 

/** StartTime event: launch another REEF job. */
@Override
public void onNext(final StartTime startTime) {

  LOG.log(Level.INFO, "Driver started: app {0} :: {1}", new Object[] {this.hostApplicationId, startTime});
  LOG.log(Level.FINE, "Launching Unmanaged AM: {0}", JAR_PATH);

  try (DriverLauncher client = DriverLauncher.getLauncher(RUNTIME_CONFIG)) {

    final String innerApplicationId = client.submit(DRIVER_CONFIG, 10000);
    LOG.log(Level.INFO, "REEF-on-REEF job submitted: host app {0} new app {1}",
        new Object[] {this.hostApplicationId, innerApplicationId});

    final Configuration yarnAmConfig = UnmanagedAmYarnDriverConfiguration.CONF
        .set(UnmanagedAmYarnDriverConfiguration.JOB_IDENTIFIER, innerApplicationId)
        .set(UnmanagedAmYarnDriverConfiguration.JOB_SUBMISSION_DIRECTORY, DRIVER_ROOT_PATH)
        .build();

    try (REEFEnvironment reef =
        REEFEnvironment.fromConfiguration(client.getUser(), yarnAmConfig, DRIVER_CONFIG)) {

      reef.run();

      final ReefServiceProtos.JobStatusProto status = reef.getLastStatus();

      LOG.log(Level.INFO, "REEF-on-REEF inner job {0} completed: state {1}",
          new Object[] {innerApplicationId, status.getState()});
    }

    LOG.log(Level.INFO,
        "REEF-on-REEF host job {0} completed: inner app {1} status {2}",
        new Object[] {this.hostApplicationId, innerApplicationId, client.getStatus()});

  } catch (final InjectionException ex) {
    LOG.log(Level.SEVERE, "REEF-on-REEF configuration error", ex);
    throw new RuntimeException("REEF-on-REEF configuration error", ex);
  }

  ThreadLogger.logThreads(LOG, Level.FINEST, "Threads running after DriverLauncher.close():");
}
 

/**
 * Set up the driver, etc. before the actual execution.
 *
 * @param args arguments.
 * @throws InjectionException     injection exception from REEF.
 * @throws ClassNotFoundException class not found exception.
 * @throws IOException            IO exception.
 */
public static void setup(final String[] args)
  throws InjectionException, ClassNotFoundException, IOException, InvalidUserMainException {
  // Get Job and Driver Confs
  LOG.info("Project Root Path: {}", Util.fetchProjectRootPath());
  builtJobConf = getJobConf(args);
  validateJobConfig(builtJobConf);

  // Registers actions for launching the DAG.
  LOG.info("Launching RPC Server");
  driverRPCServer = new DriverRPCServer();
  driverRPCServer
    .registerHandler(ControlMessage.DriverToClientMessageType.DriverStarted, event -> {
    })
    .registerHandler(ControlMessage.DriverToClientMessageType.DriverReady, event -> driverReadyLatch.countDown())
    .registerHandler(ControlMessage.DriverToClientMessageType.ExecutionDone, event -> jobDoneLatch.countDown())
    .registerHandler(ControlMessage.DriverToClientMessageType.DataCollected, message -> COLLECTED_DATA.addAll(
      SerializationUtils.deserialize(Base64.getDecoder().decode(message.getDataCollected().getData()))))
    .registerHandler(ControlMessage.DriverToClientMessageType.LaunchOptimization, message ->
      ClientUtils.handleOptimizationType(message, driverRPCServer))
    .run();

  final Configuration driverConf = getDriverConf(builtJobConf);
  final Configuration driverNcsConf = getDriverNcsConf();
  final Configuration driverMessageConfig = getDriverMessageConf();
  final String defaultExecutorResourceConfig = "[{\"type\":\"Transient\",\"memory_mb\":512,\"capacity\":5},"
    + "{\"type\":\"Reserved\",\"memory_mb\":512,\"capacity\":5}]";
  final Configuration executorResourceConfig = getJSONConf(builtJobConf, JobConf.ExecutorJSONPath.class,
    JobConf.ExecutorJSONContents.class, defaultExecutorResourceConfig);
  final Configuration bandwidthConfig = getJSONConf(builtJobConf, JobConf.BandwidthJSONPath.class,
    JobConf.BandwidthJSONContents.class, "");
  final Configuration clientConf = getClientConf();
  final Configuration schedulerConf = getSchedulerConf(builtJobConf);

  // Merge Job and Driver Confs
  jobAndDriverConf = Configurations.merge(builtJobConf, driverConf, driverNcsConf, driverMessageConfig,
    executorResourceConfig, bandwidthConfig, driverRPCServer.getListeningConfiguration(), schedulerConf);

  // Get DeployMode Conf
  deployModeConf = Configurations.merge(getDeployModeConf(builtJobConf), clientConf);

  // Start Driver and launch user program.
  if (jobAndDriverConf == null || deployModeConf == null || builtJobConf == null) {
    throw new RuntimeException("Configuration for launching driver is not ready");
  }

  // Launch driver
  LOG.info("Launching driver");
  driverReadyLatch = new CountDownLatch(1);
  jobDoneLatch = new CountDownLatch(1);
  isSetUp = true;
  driverLauncher = DriverLauncher.getLauncher(deployModeConf);
  driverLauncher.submit(jobAndDriverConf, 500);
  // When the driver is up and the resource is ready, the DriverReady message is delivered.
}