Java Code Examples for org.apache.reef.tang.Injector#getNamedInstance()

private static IRDAG compileDAG(final String[] args) throws Exception {
  final String userMainClassName;
  final String[] userMainMethodArgs;

  try {
    final Configuration configuration = JobLauncher.getJobConf(args);
    final Injector injector = Tang.Factory.getTang().newInjector(configuration);
    userMainClassName = injector.getNamedInstance(JobConf.UserMainClass.class);
    userMainMethodArgs = injector.getNamedInstance(JobConf.UserMainArguments.class).split(" ");
  } catch (final Exception e) {
    throw new RuntimeException("An exception occurred while processing configuration for invoking user main. "
      + "Note: Using compileDAG for multiple times will fail, as compileDAG method enables static method mocking "
      + "on JobLauncher and because of this Tang may misbehave afterwards.", e);
  }
  final Class userMainClass = Class.forName(userMainClassName);
  final Method userMainMethod = userMainClass.getMethod("main", String[].class);

  final ArgumentCaptor<IRDAG> captor = ArgumentCaptor.forClass(IRDAG.class);
  final ArgumentCaptor<String> stringArg = ArgumentCaptor.forClass(String.class);
  PowerMockito.mockStatic(JobLauncher.class);
  PowerMockito.doNothing().when(JobLauncher.class, "launchDAG", captor.capture(), stringArg.capture());
  userMainMethod.invoke(null, (Object) userMainMethodArgs);
  return captor.getValue();
}
 

private static DAG<IRVertex, IREdge> compileDAG(final String[] args) throws Exception {
  final String userMainClassName;
  final String[] userMainMethodArgs;

  try {
    final Configuration configuration = JobLauncher.getJobConf(args);
    final Injector injector = Tang.Factory.getTang().newInjector(configuration);
    userMainClassName = injector.getNamedInstance(JobConf.UserMainClass.class);
    userMainMethodArgs = injector.getNamedInstance(JobConf.UserMainArguments.class).split(" ");
  } catch (final Exception e) {
    throw new RuntimeException("An exception occurred while processing configuration for invoking user main. "
        + "Note: Using compileDAG for multiple times will fail, as compileDAG method enables static method mocking "
        + "on JobLauncher and because of this Tang may misbehave afterwards.", e);
  }
  final Class userMainClass = Class.forName(userMainClassName);
  final Method userMainMethod = userMainClass.getMethod("main", String[].class);

  final ArgumentCaptor<DAG> captor = ArgumentCaptor.forClass(DAG.class);
  PowerMockito.mockStatic(JobLauncher.class);
  PowerMockito.doNothing().when(JobLauncher.class, "launchDAG", captor.capture());
  userMainMethod.invoke(null, (Object) userMainMethodArgs);
  return captor.getValue();
}
 

public NetworkMessagingTestService(final String localAddress) throws InjectionException {
  // name server
  final Injector injector = Tang.Factory.getTang().newInjector();
  this.nameServer = injector.getInstance(NameServer.class);
  final Configuration netConf = NameResolverConfiguration.CONF
      .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddress)
      .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServer.getPort())
      .build();

  LOG.log(Level.FINEST, "=== Test network connection service receiver start");
  // network service for receiver
  final Injector injectorReceiver = injector.forkInjector(netConf);
  this.receiverNetworkConnService = injectorReceiver.getInstance(NetworkConnectionService.class);
  this.factory = injectorReceiver.getNamedInstance(NetworkConnectionServiceIdFactory.class);

  // network service for sender
  LOG.log(Level.FINEST, "=== Test network connection service sender start");
  final Injector injectorSender = injector.forkInjector(netConf);
  senderNetworkConnService = injectorSender.getInstance(NetworkConnectionService.class);
}
 

public static void main(final String[] args)
    throws InjectionException, BindException, IOException {

  final Tang tang = Tang.Factory.getTang();
  final JavaConfigurationBuilder cb = tang.newConfigurationBuilder();
  new CommandLine(cb)
      .registerShortNameOfClass(Local.class)
      .registerShortNameOfClass(TimeOut.class)
      .registerShortNameOfClass(OutputDir.class)
      .processCommandLine(args);

  final Injector injector = tang.newInjector(cb.build());
  final boolean isLocal = injector.getNamedInstance(Local.class);
  final String outputDir = injector.getNamedInstance(OutputDir.class);
  final int jobTimeout = injector.getNamedInstance(TimeOut.class) * 60 * 1000;

  final Configuration driverConf = getDriverConf();
  final Configuration outputServiceConf = getOutputServiceConf(isLocal, outputDir);
  final Configuration submittedConfiguration = Tang.Factory.getTang()
      .newConfigurationBuilder(driverConf, outputServiceConf)
      .build();
  final LauncherStatus state = DriverLauncher.getLauncher(getRuntimeConf(isLocal))
      .run(submittedConfiguration, jobTimeout);

  LOG.log(Level.INFO, "REEF job completed: {0}", state);
}
 

private String taskId(final Configuration partialTaskConf) {
  try {
    final Injector injector = Tang.Factory.getTang().newInjector(partialTaskConf);
    return injector.getNamedInstance(TaskConfigurationOptions.Identifier.class);
  } catch (final InjectionException e) {
    throw new RuntimeException(getQualifiedName() +
        "Injection exception while extracting taskId from partialTaskConf", e);
  }
}
 

/**
 * Create a new ContextRuntime.
 *
 * @param serviceInjector      the serviceInjector to be used.
 * @param contextConfiguration the Configuration for this context.
 * @throws ContextClientCodeException if the context cannot be instantiated.
 */
ContextRuntime(final Injector serviceInjector, final Configuration contextConfiguration,
               final Optional<ContextRuntime> parentContext) throws ContextClientCodeException {

  this.serviceInjector = serviceInjector;
  this.parentContext = parentContext;

  // Trigger the instantiation of the services
  try {

    final Set<Object> services = serviceInjector.getNamedInstance(Services.class);
    this.contextInjector = serviceInjector.forkInjector(contextConfiguration);

    this.contextLifeCycle = this.contextInjector.getInstance(ContextLifeCycle.class);

  } catch (BindException | InjectionException e) {

    final Optional<String> parentID = this.getParentContext().isPresent() ?
        Optional.of(this.getParentContext().get().getIdentifier()) :
        Optional.<String>empty();

    throw new ContextClientCodeException(
        ContextClientCodeException.getIdentifier(contextConfiguration),
        parentID, "Unable to spawn context", e);
  }

  // Trigger the context start events on contextInjector.
  this.contextLifeCycle.start();
}
 

/**
 * Fils out a JobSubmissionProto based on the driver configuration given.
 *
 * @param driverConfiguration
 * @return
 * @throws InjectionException
 * @throws IOException
 */
JobSubmissionEventImpl.Builder getJobSubmissionBuilder(final Configuration driverConfiguration)
    throws InjectionException, IOException {
  final Injector injector = Tang.Factory.getTang().newInjector(driverConfiguration);

  final boolean preserveEvaluators = injector.getNamedInstance(ResourceManagerPreserveEvaluators.class);
  final int maxAppSubmissions = injector.getNamedInstance(MaxApplicationSubmissions.class);

  final JobSubmissionEventImpl.Builder jbuilder = JobSubmissionEventImpl.newBuilder()
      .setIdentifier(returnOrGenerateDriverId(injector.getNamedInstance(DriverIdentifier.class)))
      .setDriverMemory(injector.getNamedInstance(DriverMemory.class))
      .setDriverCpuCores(injector.getNamedInstance(DriverCPUCores.class))
      .setUserName(System.getProperty("user.name"))
      .setPreserveEvaluators(preserveEvaluators)
      .setMaxApplicationSubmissions(maxAppSubmissions)
      .setConfiguration(driverConfiguration);

  for (final String globalFileName : injector.getNamedInstance(JobGlobalFiles.class)) {
    LOG.log(Level.FINEST, "Adding global file: {0}", globalFileName);
    jbuilder.addGlobalFile(getFileResourceProto(globalFileName, FileType.PLAIN));
  }

  for (final String globalLibraryName : injector.getNamedInstance(JobGlobalLibraries.class)) {
    LOG.log(Level.FINEST, "Adding global library: {0}", globalLibraryName);
    jbuilder.addGlobalFile(getFileResourceProto(globalLibraryName, FileType.LIB));
  }

  for (final String localFileName : injector.getNamedInstance(DriverLocalFiles.class)) {
    LOG.log(Level.FINEST, "Adding local file: {0}", localFileName);
    jbuilder.addLocalFile(getFileResourceProto(localFileName, FileType.PLAIN));
  }

  for (final String localLibraryName : injector.getNamedInstance(DriverLocalLibraries.class)) {
    LOG.log(Level.FINEST, "Adding local library: {0}", localLibraryName);
    jbuilder.addLocalFile(getFileResourceProto(localLibraryName, FileType.LIB));
  }

  return jbuilder;
}
 

private String contextId(final Configuration contextConf) {
  try {
    final Injector injector = Tang.Factory.getTang().newInjector(contextConf);
    return injector.getNamedInstance(ContextIdentifier.class);
  } catch (final InjectionException e) {
    throw new RuntimeException("Unable to inject context identifier from context conf", e);
  }
}
 

/**
 * Start Hello REEF job.
 *
 * @param args command line parameters.
 * @throws BindException      configuration error.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws BindException, InjectionException {

  final Tang tang = Tang.Factory.getTang();

  final JavaConfigurationBuilder cb = tang.newConfigurationBuilder();

  try{
    new CommandLine(cb)
        .registerShortNameOfClass(NodeListFilePath.class)
        .registerShortNameOfClass(SshPortNum.class)
        .processCommandLine(args);
  } catch(final IOException ex) {
    LOG.log(Level.SEVERE, "Missing parameter 'nodelist' or wrong parameter input.");
    throw new RuntimeException("Missing parameter 'nodelist' or wrong parameter input: ", ex);
  }

  final Injector injector = tang.newInjector(cb.build());

  final String nodeListFilePath = injector.getNamedInstance(NodeListFilePath.class);
  final int sshPortNum = injector.getNamedInstance(SshPortNum.class);

  final Configuration runtimeConf = getRuntimeConfiguration(nodeListFilePath, sshPortNum);
  final Configuration driverConf = getDriverConfiguration();

  final LauncherStatus status = DriverLauncher
      .getLauncher(runtimeConf)
      .run(driverConf, JOB_TIMEOUT);
  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

public static <U, T extends Name<U>> U getValue(final Configuration configuration, final Class<T> name) {
  try {
    final Injector injector = Tang.Factory.getTang().newInjector(configuration);
    return injector.getNamedInstance(name);
  } catch (InjectionException e) {
    throw new IllegalStateException(e);
  }
}
 

/**
 * Parse command line arguments and create TANG configuration ready to be submitted to REEF.
 *
 * @param args Command line arguments, as passed into main().
 * @return (immutable) TANG Configuration object.
 * @throws org.apache.reef.tang.exceptions.BindException      if configuration commandLineInjector fails.
 * @throws org.apache.reef.tang.exceptions.InjectionException if configuration commandLineInjector fails.
 * @throws java.io.IOException                                error reading the configuration.
 */
private static Configuration getClientConfiguration(final String[] args)
    throws BindException, InjectionException, IOException {

  try (LoggingScope ls = LoggingScopeFactory.getNewLoggingScope(Level.INFO, "Launch::getClientConfiguration")) {
    final Configuration commandLineConf = parseCommandLine(args);

    final Configuration clientConfiguration = ClientConfiguration.CONF
        .set(ClientConfiguration.ON_JOB_COMPLETED, JobClient.CompletedJobHandler.class)
        .set(ClientConfiguration.ON_JOB_FAILED, JobClient.FailedJobHandler.class)
        .set(ClientConfiguration.ON_RUNTIME_ERROR, JobClient.RuntimeErrorHandler.class)
        .build();

    final Injector commandLineInjector = Tang.Factory.getTang().newInjector(commandLineConf);
    final boolean isLocal = commandLineInjector.getNamedInstance(Local.class);
    final Configuration runtimeConfiguration;
    if (isLocal) {
      LOG.log(Level.INFO, "Running on the local runtime");
      runtimeConfiguration = LocalRuntimeConfiguration.CONF
          .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
          .build();
    } else {
      LOG.log(Level.INFO, "Running on YARN");
      runtimeConfiguration = YarnClientConfiguration.CONF.build();
    }

    return Tang.Factory.getTang()
        .newConfigurationBuilder(runtimeConfiguration, clientConfiguration,
            cloneCommandLineConfiguration(commandLineConf))
        .build();
  }
}
 

/**
 * Main method that starts the CLR Bridge from Java.
 *
 * @param args command line parameters.
 */
public static void main(final String[] args) {
  LOG.log(Level.INFO, "Entering Launch at :::" + new Date());
  try {
    if (args == null || args.length == 0) {
      throw new IllegalArgumentException("No arguments provided, at least a clrFolder should be supplied.");
    }
    final File dotNetFolder = new File(args[0]).getAbsoluteFile();
    final String[] removedArgs = Arrays.copyOfRange(args, 1, args.length);

    final Configuration config = getClientConfiguration(removedArgs);
    final Injector commandLineInjector = Tang.Factory.getTang().newInjector(parseCommandLine(removedArgs));
    final int waitTime = commandLineInjector.getNamedInstance(WaitTimeForDriver.class);
    final int driverMemory = commandLineInjector.getNamedInstance(DriverMemoryInMb.class);
    final boolean isLocal = commandLineInjector.getNamedInstance(Local.class);
    final String driverIdentifier = commandLineInjector.getNamedInstance(DriverIdentifier.class);
    final String jobSubmissionDirectory = commandLineInjector.getNamedInstance(DriverJobSubmissionDirectory.class);
    final boolean submit = commandLineInjector.getNamedInstance(Submit.class);
    final Injector injector = Tang.Factory.getTang().newInjector(config);
    final JobClient client = injector.getInstance(JobClient.class);
    client.setDriverInfo(driverIdentifier, driverMemory, jobSubmissionDirectory);

    if (submit) {
      client.submit(dotNetFolder, true, isLocal, null);
      client.waitForCompletion(waitTime);
    } else {
      client.submit(dotNetFolder, false, isLocal, config);
      client.waitForCompletion(0);
    }


    LOG.info("Done!");
  } catch (final BindException | InjectionException | IOException ex) {
    LOG.log(Level.SEVERE, "Job configuration error", ex);
  }
}
 

@Inject
private CommunicationGroupClientImpl(@Parameter(CommunicationGroupName.class) final String groupName,
                                    @Parameter(TaskConfigurationOptions.Identifier.class) final String taskId,
                                    @Parameter(DriverIdentifierGroupComm.class) final String driverId,
                                    final GroupCommNetworkHandler groupCommNetworkHandler,
                                    @Parameter(SerializedOperConfigs.class) final Set<String> operatorConfigs,
                                    final ConfigurationSerializer configSerializer,
                                    final NetworkService<GroupCommunicationMessage> netService,
                                    final CommGroupNetworkHandler commGroupNetworkHandler,
                                    final Injector injector) {
  this.taskId = taskId;
  this.driverId = driverId;
  LOG.finest(groupName + " has GroupCommHandler-" + groupCommNetworkHandler.toString());
  this.identifierFactory = netService.getIdentifierFactory();
  this.groupName = Utils.getClass(groupName);
  this.groupCommNetworkHandler = groupCommNetworkHandler;
  this.commGroupNetworkHandler = commGroupNetworkHandler;
  this.sender = new Sender(netService);
  this.operators = new TreeMap<>(new Comparator<Class<? extends Name<String>>>() {

    @Override
    public int compare(final Class<? extends Name<String>> o1, final Class<? extends Name<String>> o2) {
      final String s1 = o1.getSimpleName();
      final String s2 = o2.getSimpleName();
      return s1.compareTo(s2);
    }
  });
  try {
    this.groupCommNetworkHandler.register(this.groupName, commGroupNetworkHandler);

    boolean operatorIsScatterSender = false;
    for (final String operatorConfigStr : operatorConfigs) {

      final Configuration operatorConfig = configSerializer.fromString(operatorConfigStr);
      final Injector forkedInjector = injector.forkInjector(operatorConfig);

      forkedInjector.bindVolatileInstance(CommunicationGroupServiceClient.class, this);

      final GroupCommOperator operator = forkedInjector.getInstance(GroupCommOperator.class);
      final String operName = forkedInjector.getNamedInstance(OperatorName.class);
      this.operators.put(Utils.getClass(operName), operator);
      LOG.finest(operName + " has CommGroupHandler-" + commGroupNetworkHandler.toString());

      if (!operatorIsScatterSender && operator instanceof Scatter.Sender) {
        LOG.fine(operName + " is a scatter sender. Will keep track of active slave tasks.");
        operatorIsScatterSender = true;
      }
    }
    this.isScatterSender = operatorIsScatterSender;
  } catch (final InjectionException | IOException e) {
    throw new RuntimeException("Unable to deserialize operator config", e);
  }
}
 

/**
 * Start the distributed shell job.
 * @param args command line parameters.
 * @throws InjectionException configuration error.
 */
public static void main(final String[] args) throws InjectionException, IOException {

  final JavaConfigurationBuilder driverConfigBuilder = TANG.newConfigurationBuilder(STATIC_DRIVER_CONFIG);

  new CommandLine(driverConfigBuilder)
      .registerShortNameOfClass(Command.class)
      .registerShortNameOfClass(NumEvaluators.class)
      .registerShortNameOfClass(RuntimeName.class)
      .processCommandLine(args);

  final Configuration driverConfig = driverConfigBuilder.build();

  final Injector injector = TANG.newInjector(driverConfig);

  final int numEvaluators = injector.getNamedInstance(NumEvaluators.class);
  final String runtimeName = injector.getNamedInstance(RuntimeName.class);
  final String command = injector.getNamedInstance(Command.class);

  LOG.log(Level.INFO, "REEF distributed shell: {0} evaluators on {1} runtime :: {2}",
      new Object[] {numEvaluators, runtimeName, command});

  final Configuration runtimeConfig;

  switch (runtimeName) {
  case "local":
    runtimeConfig = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, numEvaluators)
        .build();
    break;
  case "yarn":
    runtimeConfig = YarnClientConfiguration.CONF.build();
    break;
  default:
    LOG.log(Level.SEVERE, "Unknown runtime: {0}", runtimeName);
    throw new IllegalArgumentException("Unknown runtime: " + runtimeName);
  }

  final LauncherStatus status = DriverLauncher.getLauncher(runtimeConfig).run(driverConfig, JOB_TIMEOUT);

  LOG.log(Level.INFO, "REEF job completed: {0}", status);
}
 

public static void main(final String[] args)
    throws InjectionException, BindException, IOException {

  final Tang tang = Tang.Factory.getTang();

  final JavaConfigurationBuilder cb = tang.newConfigurationBuilder();

  new CommandLine(cb)
      .registerShortNameOfClass(Local.class)
      .registerShortNameOfClass(TimeOut.class)
      .registerShortNameOfClass(DataLoadingREEF.InputDir.class)
      .processCommandLine(args);

  final Injector injector = tang.newInjector(cb.build());

  final boolean isLocal = injector.getNamedInstance(Local.class);
  final int jobTimeout = injector.getNamedInstance(TimeOut.class) * 60 * 1000;
  final String inputDir = injector.getNamedInstance(DataLoadingREEF.InputDir.class);

  final Configuration runtimeConfiguration;
  if (isLocal) {
    LOG.log(Level.INFO, "Running Data Loading demo on the local runtime");
    runtimeConfiguration = LocalRuntimeConfiguration.CONF
        .set(LocalRuntimeConfiguration.MAX_NUMBER_OF_EVALUATORS, MAX_NUMBER_OF_EVALUATORS)
        .build();
  } else {
    LOG.log(Level.INFO, "Running Data Loading demo on YARN");
    runtimeConfiguration = YarnClientConfiguration.CONF.build();
  }

  final EvaluatorRequest computeRequest = EvaluatorRequest.newBuilder()
      .setNumber(NUM_COMPUTE_EVALUATORS)
      .setMemory(512)
      .setNumberOfCores(1)
      .build();

  final EvaluatorRequest dataRequest = EvaluatorRequest.newBuilder()
      .setMemory(512)
      .setNumberOfCores(1)
      .build();

  final Configuration dataLoadConfiguration = new DataLoadingRequestBuilder()
      .setInputFormatClass(TextInputFormat.class)
      .setInputPath(inputDir)
      .setNumberOfDesiredSplits(NUM_SPLITS)
      .addComputeRequest(computeRequest)
      .addDataRequest(dataRequest)
      .setDriverConfigurationModule(DriverConfiguration.CONF
          .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(LineCounter.class))
          .set(DriverConfiguration.ON_CONTEXT_ACTIVE, LineCounter.ContextActiveHandler.class)
          .set(DriverConfiguration.ON_TASK_COMPLETED, LineCounter.TaskCompletedHandler.class)
          .set(DriverConfiguration.DRIVER_IDENTIFIER, "DataLoadingREEF"))
      .build();

  final LauncherStatus state =
      DriverLauncher.getLauncher(runtimeConfiguration).run(dataLoadConfiguration, jobTimeout);

  LOG.log(Level.INFO, "REEF job completed: {0}", state);
}
 

/**
 * Checks if profiling is enabled.
 *
 * @param injector the tang injector that stores value of ProfilingEnabled.
 * @return true if profiling is enabled
 * @throws InjectionException if name resolution fails
 */
public static boolean isProfilingEnabled(final Injector injector) throws InjectionException {
  return injector.getNamedInstance(getProfilingEnabledClass());
}