org.apache.reef.tang.Configuration Java Examples

/**
 * Run the AddOne test.
 */
@Test
public void testVortexAddOne() {

  final Configuration vortexMasterConf = VortexMasterConf.CONF
      .set(VortexMasterConf.WORKER_NUM, 2)
      .set(VortexMasterConf.WORKER_MEM, 64)
      .set(VortexMasterConf.WORKER_CORES, 4)
      .set(VortexMasterConf.WORKER_CAPACITY, 2000)
      .set(VortexMasterConf.VORTEX_START, AddOneTestStart.class)
      .build();

  final VortexJobConf vortexJobConf = VortexJobConf.newBuilder()
      .setJobName("TEST_Vortex_AddOneTest")
      .setVortexMasterConf(vortexMasterConf)
      .build();

  final LauncherStatus status = this.testEnvironment.run(vortexJobConf.getConfiguration());
  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 

/**
 * Tests that an evaluator's state is changed to closing and closed in order during closing.
 */
@Test
public void testEvaluatorClosingState() throws InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES,
          EnvironmentUtils.getClassLocation(EvaluatorCloseDriver.class))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_EvaluatorCloseTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, EvaluatorCloseDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, EvaluatorCloseDriver.TaskRunningHandler.class)
      .set(DriverConfiguration.ON_DRIVER_STOP, EvaluatorCloseDriver.StopHandler.class)
      .build();

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("The result state of evaluator closing state test is " + status,
      status.isSuccess());
}
 

@Test
public void setClassTest() throws BindException, InjectionException {
  final Configuration c = SetClassConfigurationModule.CONF
      .set(SetClassConfigurationModule.P, SubA.class)
      .set(SetClassConfigurationModule.P, SubB.class)
      .build();
  final Set<Super> s = Tang.Factory.getTang().newInjector(c).getNamedInstance(SetClass.class);
  Assert.assertEquals(2, s.size());
  boolean sawA = false, sawB = false;
  for (final Super sup : s) {
    if (sup instanceof SubA) {
      sawA = true;
    } else if (sup instanceof SubB) {
      sawB = true;
    } else {
      Assert.fail();
    }
  }
  Assert.assertTrue(sawA && sawB);
}
 

/**
* Tests whether the runtime passes the rack information to the driver.
* The success scenario is if it receives the default rack, fails otherwise.
*/
@Test
public void testRackAwareEvaluatorRunningOnDefaultRack() {
  //Given
  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_RackAwareEvaluator")
      .set(DriverConfiguration.GLOBAL_LIBRARIES,
          EnvironmentUtils.getClassLocation(RackAwareEvaluatorTestDriver.class))
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, RackAwareEvaluatorTestDriver.EvaluatorAllocatedHandler.class)
      .build();

  // When
  final LauncherStatus status = this.testEnvironment.run(driverConfiguration);
  // Then
  Assert.assertTrue("Job state after execution: " + status, status.isSuccess());
}
 

/**
 * Instantiate REEF DriverServiceLauncher. This method is called from REEFLauncher.main().
 * @param clockConfigPath Path to the local file that contains serialized configuration
 *                        for the driver client.
 * @return An instance of the configured REEFLauncher object.
 */
private static JavaDriverClientLauncher getLauncher(final String clockConfigPath, final int driverServicePort) {
  try {
    final Configuration clockArgConfig = Configurations.merge(
        LAUNCHER_STATIC_CONFIG,
        DriverClientGrpcConfiguration.CONF
            .set(DriverClientGrpcConfiguration.DRIVER_SERVICE_PORT, driverServicePort)
            .build(),
        TANG.newConfigurationBuilder()
            .bindNamedParameter(ClockConfigurationPath.class, clockConfigPath)
            .build());

    return TANG.newInjector(clockArgConfig).getInstance(JavaDriverClientLauncher.class);
  } catch (final InjectionException ex) {
    throw fatal("Unable to instantiate REEFLauncher.", ex);
  }
}
 

private static IRDAG compileDAG(final String[] args) throws Exception {
  final String userMainClassName;
  final String[] userMainMethodArgs;

  try {
    final Configuration configuration = JobLauncher.getJobConf(args);
    final Injector injector = Tang.Factory.getTang().newInjector(configuration);
    userMainClassName = injector.getNamedInstance(JobConf.UserMainClass.class);
    userMainMethodArgs = injector.getNamedInstance(JobConf.UserMainArguments.class).split(" ");
  } catch (final Exception e) {
    throw new RuntimeException("An exception occurred while processing configuration for invoking user main. "
      + "Note: Using compileDAG for multiple times will fail, as compileDAG method enables static method mocking "
      + "on JobLauncher and because of this Tang may misbehave afterwards.", e);
  }
  final Class userMainClass = Class.forName(userMainClassName);
  final Method userMainMethod = userMainClass.getMethod("main", String[].class);

  final ArgumentCaptor<IRDAG> captor = ArgumentCaptor.forClass(IRDAG.class);
  final ArgumentCaptor<String> stringArg = ArgumentCaptor.forClass(String.class);
  PowerMockito.mockStatic(JobLauncher.class);
  PowerMockito.doNothing().when(JobLauncher.class, "launchDAG", captor.capture(), stringArg.capture());
  userMainMethod.invoke(null, (Object) userMainMethodArgs);
  return captor.getValue();
}
 

public static void main(final String[] args) throws Exception {
  final Tang tang = Tang.Factory.getTang();
  final JavaConfigurationBuilder cb = tang.newConfigurationBuilder();
  final CommandLine cl = new CommandLine(cb);
  cl.registerShortNameOfClass(Timer.Seconds.class);
  cl.processCommandLine(args);
  final Configuration conf = cb.build();
  System.out.println("start conf");
  final AvroConfigurationSerializer avroSerializer = new AvroConfigurationSerializer();
  System.out.println(avroSerializer.toString(conf));
  System.out.println("end conf");
  final InjectorImpl injector = (InjectorImpl) tang.newInjector(conf);
  final InjectionPlan<Timer> ip = injector.getInjectionPlan(Timer.class);
  System.out.println(ip.toPrettyString());
  System.out.println("Number of plans:" + ip.getNumAlternatives());
  final Timer timer = injector.getInstance(Timer.class);
  System.out.println("Tick...");
  timer.sleep();
  System.out.println("Tock.");
}
 

@Test
public void testRogueThread() throws InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();
  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "Test_RogueThreadTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, OnDriverStartedAllocateOne.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, RogueThreadDriver.EvaluatorAllocationHandler.class)
      .set(DriverConfiguration.ON_TASK_FAILED, ExpectedTaskFailureHandler.class)
      .build();
  final LauncherStatus state = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(driverConfiguration, this.testEnvironment.getTestTimeout());

  Assert.assertTrue("Job state after execution: " + state, state.isSuccess());


}
 

/**
 * Launch the vortex job, passing appropriate arguments.
 */
public static void main(final String[] args) {
  final Configuration vortexMasterConf = VortexMasterConf.CONF
      .set(VortexMasterConf.WORKER_NUM, 1)
      .set(VortexMasterConf.WORKER_MEM, 1024)
      .set(VortexMasterConf.WORKER_CORES, 1)
      .set(VortexMasterConf.WORKER_CAPACITY, 2000)
      .set(VortexMasterConf.VORTEX_START, HelloVortexStart.class)
      .build();

  final VortexJobConf vortexJobConf = VortexJobConf.newBuilder()
      .setVortexMasterConf(vortexMasterConf)
      .setJobName("HelloVortex")
      .build();

  VortexLauncher.launchLocal(vortexJobConf);
}
 

private Configuration getDriverConfiguration() {
  final Configuration driverConf = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "WatcherTest")
      .set(DriverConfiguration.ON_DRIVER_STARTED, WatcherTestDriver.DriverStartedHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, WatcherTestDriver.EvaluatorAllocatedHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_FAILED, WatcherTestDriver.EvaluatorFailedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_ACTIVE, WatcherTestDriver.ContextActivatedHandler.class)
      .set(DriverConfiguration.ON_CONTEXT_FAILED, WatcherTestDriver.ContextFailedHandler.class)
      .set(DriverConfiguration.ON_TASK_FAILED, WatcherTestDriver.TaskFailedHandler.class)
      .set(DriverConfiguration.ON_TASK_RUNNING, WatcherTestDriver.TaskRunningHandler.class)
      .set(DriverConfiguration.ON_TASK_SUSPENDED, WatcherTestDriver.TaskSuspendedHandler.class)
      .build();

  final Configuration runtimeStopConf = Tang.Factory.getTang().newConfigurationBuilder()
      .bindSetEntry(Clock.RuntimeStopHandler.class, WatcherTestDriver.RuntimeStopHandler.class)
      .build();

  return Configurations.merge(driverConf, runtimeStopConf, WatcherConfiguration.CONF
      .set(WatcherConfiguration.EVENT_STREAMS, LogEventStream.class)
      .set(WatcherConfiguration.EVENT_STREAMS, TestEventStream.class)
      .build());
}
 

private Injector createNameClientInjector() {
  try {
    final Configuration configuration = TANG.newConfigurationBuilder()
        .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class)
        .build();
    final Injector injector = TANG.newInjector(configuration);
    final LocalAddressProvider localAddressProvider = injector.getInstance(LocalAddressProvider.class);
    final NameServer nameServer = injector.getInstance(NameServer.class);
    final Configuration nameClientConfiguration = NameResolverConfiguration.CONF
        .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddressProvider.getLocalAddress())
        .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServer.getPort())
        .build();
    return injector.forkInjector(nameClientConfiguration);
  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }
}
 

private LauncherStatus runEvaluatorSizeTest(final int megaBytes) throws BindException, InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();

  final Configuration testConfiguration = EvaluatorSizeTestConfiguration.CONF
      .set(EvaluatorSizeTestConfiguration.MEMORY_SIZE, 777)
      .build();

  final Configuration driverConfiguration = DriverConfiguration.CONF
      .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "TEST_EvaluatorSizeTest-" + megaBytes)
      .set(DriverConfiguration.ON_DRIVER_STARTED, EvaluatorSizeTestDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED,
          EvaluatorSizeTestDriver.EvaluatorAllocatedHandler.class).build();

  final Configuration mergedDriverConfiguration = Tang.Factory.getTang()
      .newConfigurationBuilder(driverConfiguration, testConfiguration).build();

  final LauncherStatus state = DriverLauncher.getLauncher(runtimeConfiguration)
      .run(mergedDriverConfiguration, this.testEnvironment.getTestTimeout());
  return state;
}
 

public static LauncherStatus run(
    final Class<? extends Task> failTaskClass,
    final Configuration runtimeConfig,
    final ClientProtocol.DriverClientConfiguration driverClientConfiguration,
    final int timeOut) throws InjectionException, IOException {

  final Configuration driverConfig = DriverClientConfiguration.CONF
      .set(DriverClientConfiguration.ON_EVALUATOR_ALLOCATED, Driver.AllocatedEvaluatorHandler.class)
      .set(DriverClientConfiguration.ON_TASK_RUNNING, Driver.RunningTaskHandler.class)
      .set(DriverClientConfiguration.ON_CONTEXT_ACTIVE, Driver.ActiveContextHandler.class)
      .set(DriverClientConfiguration.ON_DRIVER_STARTED, Driver.StartHandler.class)
      .build();

  final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder();
  cb.addConfiguration(driverConfig);
  cb.bindNamedParameter(Driver.FailTaskName.class, failTaskClass.getSimpleName());

  final Configuration driverServiceConfiguration =
      FailBridgeClientUtils.setupDriverService(
          runtimeConfig,
          cb.build(),
          driverClientConfiguration);
  return TestDriverLauncher.getLauncher(runtimeConfig).run(driverServiceConfiguration, timeOut);
}
 

private LauncherStatus run(final long jobTimeout, final long statusTimeout) {

    final long startTime = System.currentTimeMillis();
    LOG.log(Level.INFO, "Submitting REEF Job");

    final Configuration driverConfig = DriverConfiguration.CONF
        .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(this.getClass()))
        .set(DriverConfiguration.DRIVER_IDENTIFIER, "DriverMessagingTest")
        .set(DriverConfiguration.ON_DRIVER_STARTED, DriverMessagingDriver.StartHandler.class)
        .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, DriverMessagingDriver.AllocatedEvaluatorHandler.class)
        .set(DriverConfiguration.ON_CLIENT_MESSAGE, DriverMessagingDriver.ClientMessageHandler.class)
        .build();


    this.reef.submit(driverConfig);

    synchronized (this) {
      while (!this.status.isDone()) {
        LOG.log(Level.INFO, "Waiting for REEF job to finish.");
        try {
          this.wait(statusTimeout);
        } catch (final InterruptedException ex) {
          LOG.log(Level.FINER, "Waiting for REEF job interrupted.", ex);
        }
        if (System.currentTimeMillis() - startTime >= jobTimeout) {
          LOG.log(Level.INFO, "Waiting for REEF job timed out after {0} sec.",
              (System.currentTimeMillis() - startTime) / 1000);
          break;
        }
      }
    }

    this.reef.close();
    synchronized (this) {
      return this.status;
    }
  }
 

private static Optional<String> getUserBoundJobSubmissionDirectory(final Configuration configuration) {
  try {
    return Optional.ofNullable(Tang.Factory.getTang().newInjector(configuration)
        .getNamedInstance(DriverJobSubmissionDirectory.class));
  } catch (final InjectionException ex) {
    return Optional.empty();
  }

}
 

/**
 * @return the configuration of the runtime
 */
private static Configuration getRuntimeConfiguration(final String nodeListFileName, final int sshPortNum) {
  return StandaloneRuntimeConfiguration.CONF
      .set(StandaloneRuntimeConfiguration.NODE_LIST_FILE_PATH, nodeListFileName)
      .set(StandaloneRuntimeConfiguration.SSH_PORT_NUM, sshPortNum)
      .build();
}
 

/**
 * Assembles the Driver configuration.
 *
 * @return the Driver configuration.
 */
private Configuration getDriverConfiguration() {
  return DriverConfiguration.CONF
      .set(DriverConfiguration.DRIVER_IDENTIFIER, "ConfigurationProviderTest")
      .set(DriverConfiguration.GLOBAL_LIBRARIES,
          EnvironmentUtils.getClassLocation(ConfigurationProviderTestDriver.class))
      .set(DriverConfiguration.ON_DRIVER_STARTED, ConfigurationProviderTestDriver.StartHandler.class)
      .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED,
          ConfigurationProviderTestDriver.EvaluatorAllocatedHandler.class)
      .build();
}
 

public static Configuration getYarnConfiguration() {
  final Configuration yarnDriverRestartConfiguration = YarnDriverRestartConfiguration.CONF
      .build();

  final Configuration driverRestartHandlerConfigurations = DriverRestartConfiguration.CONF
      .set(DriverRestartConfiguration.ON_DRIVER_RESTARTED,
          ReefEventStateManager.DriverRestartHandler.class)
      .set(DriverRestartConfiguration.ON_DRIVER_RESTART_TASK_RUNNING,
          ReefEventStateManager.DriverRestartTaskRunningStateHandler.class)
      .set(DriverRestartConfiguration.ON_DRIVER_RESTART_CONTEXT_ACTIVE,
          ReefEventStateManager.DriverRestartActiveContextStateHandler.class)
      .build();

  return Configurations.merge(yarnDriverRestartConfiguration, driverRestartHandlerConfigurations);
}
 

@Override
public synchronized Configuration getRuntimeConfiguration() {
  assert this.ready;
  try {
    Configuration userConfiguration = AzureBatchRuntimeConfiguration.fromEnvironment();
    final Injector injector = Tang.Factory.getTang().newInjector(userConfiguration);
    final AzureBatchRuntimeConfigurationProvider runtimeConfigurationProvider =
        injector.getInstance(AzureBatchRuntimeConfigurationProvider.class);
    return runtimeConfigurationProvider.getAzureBatchRuntimeConfiguration();
  } catch (IOException | InjectionException e) {
    throw new RuntimeException(e);
  }
}
 

/**
 * Start Hello REEF job. Runs method runHelloCLR().
 *
 * @param args command line parameters.
 * @throws org.apache.reef.tang.exceptions.BindException      configuration error.
 * @throws org.apache.reef.tang.exceptions.InjectionException configuration error.
 */
public static void main(final String[] args) throws BindException, InjectionException {
  final Configuration runtimeConfiguration = LocalRuntimeConfiguration.CONF
      .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, 2)
      .build();

  final File dotNetFolder = new File(args[0]).getAbsoluteFile();
  final LauncherStatus status = runHelloCLR(runtimeConfiguration, JOB_TIMEOUT, dotNetFolder);
  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

/**
 * Assembles the Driver configuration.
 *
 * @param jobFolder the job folder.
 * @param clientRemoteId the client remote id.
 * @param jobId the job id.
 * @param applicationConfiguration the application configuration.
 * @return the Driver configuration.
 */
@Override
public Configuration getDriverConfiguration(final URI jobFolder,
                                            final String clientRemoteId,
                                            final String jobId,
                                            final Configuration applicationConfiguration) {
  ConfigurationModuleBuilder driverConfigurationBuilder = AzureBatchDriverConfiguration.CONF.getBuilder()
      .bindImplementation(CommandBuilder.class, this.commandBuilder.getClass());

  // If using docker containers, then use a different set of bindings
  if (this.containerRegistryProvider.isValid()) {
    driverConfigurationBuilder = driverConfigurationBuilder
        .bindImplementation(LocalAddressProvider.class, ContainerBasedLocalAddressProvider.class)
        .bindImplementation(TcpPortProvider.class, SetTcpPortProvider.class);
  }

  final Configuration driverConfiguration = driverConfigurationBuilder.build()
      .set(AzureBatchDriverConfiguration.JOB_IDENTIFIER, jobId)
      .set(AzureBatchDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId)
      .set(AzureBatchDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack)
      .set(AzureBatchDriverConfiguration.RUNTIME_NAME, RuntimeIdentifier.RUNTIME_NAME)
      .set(AzureBatchDriverConfiguration.AZURE_BATCH_ACCOUNT_URI, this.azureBatchAccountUri)
      .set(AzureBatchDriverConfiguration.AZURE_BATCH_ACCOUNT_NAME, this.azureBatchAccountName)
      .set(AzureBatchDriverConfiguration.AZURE_BATCH_POOL_ID, this.azureBatchPoolId)
      .set(AzureBatchDriverConfiguration.AZURE_STORAGE_ACCOUNT_NAME, this.azureStorageAccountName)
      .set(AzureBatchDriverConfiguration.AZURE_STORAGE_CONTAINER_NAME, this.azureStorageContainerName)
      .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_SERVER,
          this.containerRegistryProvider.getContainerRegistryServer())
      .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_USERNAME,
          this.containerRegistryProvider.getContainerRegistryUsername())
      .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_PASSWORD,
          this.containerRegistryProvider.getContainerRegistryPassword())
      .set(AzureBatchDriverConfiguration.CONTAINER_IMAGE_NAME,
          this.containerRegistryProvider.getContainerImageName())
      .setMultiple(AzureBatchDriverConfiguration.TCP_PORT_SET, this.tcpPortSet)
      .build();
  return Configurations.merge(driverConfiguration, applicationConfiguration);
}
 

@Override
public Configuration getContextConfiguration() {
  LOG.entering("GroupCommDriverImpl", "getContextConf");
  final Configuration retVal = ContextConfiguration.CONF.set(ContextConfiguration.IDENTIFIER,
      "GroupCommunicationContext-" + contextIds.getAndIncrement()).build();
  LOG.exiting("GroupCommDriverImpl", "getContextConf", confSerializer.toString(retVal));
  return retVal;
}
 

@Override
public Configuration roundTrip(final Configuration configuration, final ClassHierarchy classHierarchy)
    throws Exception {
  final File tempFile = File.createTempFile("TangTest", "avroconf");
  final AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
  serializer.toTextFile(configuration, tempFile);
  final Configuration c = serializer.fromTextFile(tempFile, classHierarchy);
  tempFile.delete();
  return c;
}
 

/**
 * Run the Task scheduler. If '-retain true' option is passed via command line,
 * the scheduler reuses evaluators to submit new Tasks.
 * @param runtimeConf The runtime configuration (e.g. Local, YARN, etc)
 * @param args Command line arguments.
 * @throws InjectionException
 * @throws java.io.IOException
 */
public static void runTaskScheduler(final Configuration runtimeConf, final String[] args)
    throws InjectionException, IOException, ParseException {
  final Tang tang = Tang.Factory.getTang();

  final Configuration commandLineConf = CommandLine.parseToConfiguration(args, Retain.class);

  // Merge the configurations to run Driver
  final Configuration driverConf = Configurations.merge(getDriverConf(), getHttpConf(), commandLineConf);

  final REEF reef = tang.newInjector(runtimeConf).getInstance(REEF.class);
  reef.submit(driverConf);
}
 

@Override
public Configuration roundTrip(final Configuration configuration, final ClassHierarchy classHierarchy)
    throws Exception {
  final File tempFile = java.io.File.createTempFile("TangTest", "txt");
  final ConfigurationSerializer serializer = new AvroConfigurationSerializer();
  serializer.toTextFile(configuration, tempFile);
  final Configuration conf = serializer.fromTextFile(tempFile, classHierarchy);
  tempFile.delete();
  return conf;
}
 

@Test
public void testDriverCompleted() throws BindException, InjectionException {
  final Configuration runtimeConfiguration = this.testEnvironment.getRuntimeConfiguration();
  // FailDriverTest can be replaced with any other class never used in FailDriver
  final LauncherStatus status = FailClient.runClient(
      FailDriverTest.class, runtimeConfiguration, this.testEnvironment.getTestTimeout());
  Assert.assertEquals(LauncherStatus.COMPLETED, status);
}
 

@Override
public void onNext(final JobSubmissionEvent jobSubmissionEvent) {

  final String id = jobSubmissionEvent.getIdentifier();
  LOG.log(Level.FINEST, "Submitting{0} job: {1}",
      new Object[] {this.isUnmanaged ? " UNMANAGED AM" : "", jobSubmissionEvent});

  try (YarnSubmissionHelper submissionHelper = new YarnSubmissionHelper(this.yarnConfiguration,
      this.fileNames, this.classpath, this.yarnProxyUser, this.tokenProvider, this.isUnmanaged)) {

    LOG.log(Level.FINE, "Assembling submission JAR for the Driver.");
    final Optional<String> userBoundJobSubmissionDirectory =
        getUserBoundJobSubmissionDirectory(jobSubmissionEvent.getConfiguration());
    final JobFolder jobFolderOnDfs = userBoundJobSubmissionDirectory.isPresent()
        ? this.uploader.createJobFolder(userBoundJobSubmissionDirectory.get())
        : this.uploader.createJobFolder(submissionHelper.getApplicationId());
    final Configuration driverConfiguration = makeDriverConfiguration(jobSubmissionEvent, jobFolderOnDfs.getPath());
    final File jobSubmissionFile = this.jobJarMaker.createJobSubmissionJAR(jobSubmissionEvent, driverConfiguration);
    final LocalResource driverJarOnDfs =
        jobFolderOnDfs.uploadAsLocalResource(jobSubmissionFile, LocalResourceType.ARCHIVE);

    submissionHelper
        .addLocalResource(this.fileNames.getREEFFolderName(), driverJarOnDfs)
        .setApplicationName(id)
        .setDriverResources(jobSubmissionEvent.getDriverMemory().get(), jobSubmissionEvent.getDriverCPUCores().get())
        .setPriority(getPriority(jobSubmissionEvent))
        .setQueue(getQueue(jobSubmissionEvent))
        .setPreserveEvaluators(getPreserveEvaluators(jobSubmissionEvent))
        .setMaxApplicationAttempts(getMaxApplicationSubmissions(jobSubmissionEvent))
        .submit();

    this.applicationId = submissionHelper.getStringApplicationId();
    LOG.log(Level.FINEST, "Submitted{0} job with ID {1} :: {2}", new String[] {
        this.isUnmanaged ? " UNMANAGED AM" : "", id, this.applicationId});

  } catch (final YarnException | IOException e) {
    throw new RuntimeException("Unable to submit Driver to YARN: " + id, e);
  }
}
 

public static LauncherStatus runClient(
    final Class<?> failMsgClass,
    final Configuration runtimeConfig,
    final int timeOut) throws IOException, InjectionException {
  ClientProtocol.DriverClientConfiguration.Builder builder =
      ClientProtocol.DriverClientConfiguration.newBuilder()
      .setJobid("FailBridge_" + failMsgClass.getSimpleName())
      .addGlobalLibraries(EnvironmentUtils.getClassLocation(FailDriver.class));
  builder.setOperatingSystem(
      OSUtils.isWindows() ?
          ClientProtocol.DriverClientConfiguration.OS.WINDOWS :
          ClientProtocol.DriverClientConfiguration.OS.LINUX);

  return runClient(failMsgClass, runtimeConfig, builder.build(), timeOut);
}
 

@Override
public void toFile(final Configuration conf, final File file) throws IOException {
  final AvroConfiguration avroConfiguration = toAvro(conf);
  final DatumWriter<AvroConfiguration> configurationWriter = new SpecificDatumWriter<>(AvroConfiguration.class);
  try (DataFileWriter<AvroConfiguration> dataFileWriter = new DataFileWriter<>(configurationWriter)) {
    dataFileWriter.create(avroConfiguration.getSchema(), file);
    dataFileWriter.append(avroConfiguration);
  }
}
 

public static LauncherStatus run(final Configuration runtimeConfiguration,
                                 final int launcherTimeout) throws BindException, InjectionException {

  final Configuration clientConfiguration = ClientConfiguration.CONF
      .set(ClientConfiguration.ON_JOB_RUNNING, DriverMessaging.RunningJobHandler.class)
      .set(ClientConfiguration.ON_JOB_MESSAGE, DriverMessaging.JobMessageHandler.class)
      .set(ClientConfiguration.ON_JOB_COMPLETED, DriverMessaging.CompletedJobHandler.class)
      .set(ClientConfiguration.ON_JOB_FAILED, DriverMessaging.FailedJobHandler.class)
      .set(ClientConfiguration.ON_RUNTIME_ERROR, DriverMessaging.RuntimeErrorHandler.class)
      .build();

  return Tang.Factory.getTang()
      .newInjector(runtimeConfiguration, clientConfiguration)
      .getInstance(DriverMessaging.class).run(launcherTimeout, 1000);
}