org.apache.reef.driver.context.ContextConfiguration Java Examples

private Configuration getExecutorConfiguration(final String executorId, final int executorCapacity) {
  final Configuration executorConfiguration = JobConf.EXECUTOR_CONF
      .set(JobConf.EXECUTOR_ID, executorId)
      .set(JobConf.EXECUTOR_CAPACITY, executorCapacity)
      .set(JobConf.GLUSTER_DISK_DIRECTORY, glusterDirectory)
      .set(JobConf.LOCAL_DISK_DIRECTORY, localDirectory)
      .set(JobConf.JOB_ID, jobId)
      .build();

  final Configuration contextConfiguration = ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, executorId) // We set: contextId = executorId
      .set(ContextConfiguration.ON_CONTEXT_STARTED, NemoContext.ContextStartHandler.class)
      .set(ContextConfiguration.ON_CONTEXT_STOP, NemoContext.ContextStopHandler.class)
      .build();

  final Configuration ncsConfiguration =  getExecutorNcsConfiguration();
  final Configuration messageConfiguration = getExecutorMessageConfiguration(executorId);

  return Configurations.merge(executorConfiguration, contextConfiguration, ncsConfiguration, messageConfiguration);
}
 

/**
 * Uses the AllocatedEvaluator to launch a JVM task.
 *
 * @param allocatedEvaluator
 */
void onNextJVM(final AllocatedEvaluator allocatedEvaluator) {
  try {
    final Configuration contextConfiguration = ContextConfiguration.CONF
        .set(ContextConfiguration.IDENTIFIER, "HelloREEFContext")
        .build();

    final Configuration taskConfiguration = TaskConfiguration.CONF
        .set(TaskConfiguration.IDENTIFIER, "HelloREEFTask")
        .set(TaskConfiguration.TASK, HelloTask.class)
        .build();

    allocatedEvaluator.submitContextAndTask(contextConfiguration, taskConfiguration);
  } catch (final BindException ex) {
    final String message = "Unable to setup Task or Context configuration.";
    LOG.log(Level.SEVERE, message, ex);
    throw new RuntimeException(message, ex);
  }
}
 

/**
 * Uses the AllocatedEvaluator to launch a CLR task.
 *
 * @param allocatedEvaluator
 */
void onNextCLR(final AllocatedEvaluator allocatedEvaluator) {
  try {
    allocatedEvaluator.setProcess(clrProcessFactory.newEvaluatorProcess());
    final Configuration contextConfiguration = ContextConfiguration.CONF
        .set(ContextConfiguration.IDENTIFIER, "HelloREEFContext")
        .build();

    final Configuration taskConfiguration = getCLRTaskConfiguration("Hello_From_CLR");

    allocatedEvaluator.submitContextAndTask(contextConfiguration, taskConfiguration);
  } catch (final BindException ex) {
    final String message = "Unable to setup Task or Context configuration.";
    LOG.log(Level.SEVERE, message, ex);
    throw new RuntimeException(message, ex);
  }
}
 

@Override
public void onNext(final AllocatedEvaluator eval) {
  try {

    LOG.log(Level.INFO, "Allocated Evaluator: {0}", eval.getId());

    final Configuration thisContextConfiguration = ContextConfiguration.CONF.set(
        ContextConfiguration.IDENTIFIER, eval.getId() + "_context").build();

    eval.submitContext(Tang.Factory.getTang()
        .newConfigurationBuilder(thisContextConfiguration, contextConfig).build());

  } catch (final BindException ex) {
    throw new RuntimeException(ex);
  }
}
 

@Override
public void onNext(final AllocatedEvaluator allocatedEvaluator) {

  LOG.log(Level.FINE, "Submitting root context");

  try {

    final Configuration contextConfiguration = ContextConfiguration.CONF
        .set(ContextConfiguration.ON_CONTEXT_STARTED, ContextStartHandler1.class)
        .set(ContextConfiguration.ON_CONTEXT_STOP, ContextStopHandler1.class)
        .set(ContextConfiguration.IDENTIFIER, CONTEXT_1_IDENTIFIER)
        .build();

    allocatedEvaluator.submitContext(contextConfiguration);

    synchronized (SubContextDriver.this) {
      SubContextDriver.this.state = State.CONTEXT_1_SUBMITTED;
    }

  } catch (final BindException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public void onNext(final AllocatedEvaluator eb) {
  try {
    final Configuration contextConfiguration = ContextConfiguration.CONF
        .set(ContextConfiguration.IDENTIFIER, "StatePassingContext")
        .build();

    final Configuration serviceConfiguration = ServiceConfiguration.CONF
        .set(ServiceConfiguration.SERVICES, Counter.class)
        .build();

    eb.submitContextAndService(contextConfiguration, serviceConfiguration);
  } catch (final BindException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public void onNext(final AllocatedEvaluator eval) {

  try {

    LOG.log(Level.INFO, "Submit task: Fail2");

    final Configuration contextConfig = ContextConfiguration.CONF
        .set(ContextConfiguration.IDENTIFIER, "Fail2")
        .build();

    final Configuration taskConfig = TaskConfiguration.CONF
        .set(TaskConfiguration.IDENTIFIER, "Fail2")
        .set(TaskConfiguration.TASK, FailTaskCall.class)
        .build();

    eval.submitContextAndTask(contextConfig, taskConfig);

  } catch (final BindException ex) {
    LOG.log(Level.WARNING, "Configuration error", ex);
    throw new RuntimeException(ex);
  }
}
 

private Configuration getExecutorConfiguration(final String executorId) {
  final Configuration executorConfiguration = JobConf.EXECUTOR_CONF
    .set(JobConf.EXECUTOR_ID, executorId)
    .set(JobConf.GLUSTER_DISK_DIRECTORY, glusterDirectory)
    .set(JobConf.LOCAL_DISK_DIRECTORY, localDirectory)
    .set(JobConf.JOB_ID, jobId)
    .build();

  final Configuration contextConfiguration = ContextConfiguration.CONF
    .set(ContextConfiguration.IDENTIFIER, executorId) // We set: contextId = executorId
    .set(ContextConfiguration.ON_CONTEXT_STARTED, NemoContext.ContextStartHandler.class)
    .set(ContextConfiguration.ON_CONTEXT_STOP, NemoContext.ContextStopHandler.class)
    .build();

  final Configuration ncsConfiguration = getExecutorNcsConfiguration();
  final Configuration messageConfiguration = getExecutorMessageConfiguration(executorId);
  final Configuration dataPlaneConfiguration = dataPlaneConf.getDataPlaneConfiguration();

  return Configurations.merge(executorConfiguration, contextConfiguration, ncsConfiguration,
    messageConfiguration, dataPlaneConfiguration);
}
 

@Override
public void onNext(final AllocatedEvaluator allocatedEvaluator) {
  final String evalId = allocatedEvaluator.getId();
  LOG.log(Level.FINE, "Got allocated evaluator: {0}", evalId);
  if (numEvaluatorsLeftToSubmit.getAndDecrement() > 0) {
    LOG.log(Level.FINE, "Submitting poisoned context. {0} to go.", numEvaluatorsLeftToSubmit);
    allocatedEvaluator.submitContext(
        Tang.Factory.getTang()
            .newConfigurationBuilder(
                ContextConfiguration.CONF
                    .set(ContextConfiguration.IDENTIFIER, "Poisoned Context: " + evalId)
                    .build(),
                PoisonedConfiguration.CONTEXT_CONF
                    .set(PoisonedConfiguration.CRASH_PROBABILITY, "1")
                    .set(PoisonedConfiguration.CRASH_TIMEOUT, "1")
                    .build())
            .build());
  } else {
    LOG.log(Level.FINE, "Closing evaluator {0}", evalId);
    allocatedEvaluator.close();
    FailureDriver.this.numEvaluatorsLeftToClose.decrementAndGet();
  }
}
 

@Override
public void onNext(final ActiveContext activeContext) {
  if (activeContext.getId().equals(ROOT_CONTEXT_ID)) {
    activeContext.submitContext(ContextConfiguration.CONF
        .set(ContextConfiguration.IDENTIFIER, FIRST_CONTEXT_ID)
        .build());
  } else if (activeContext.getId().equals(FIRST_CONTEXT_ID)) {
    activeContext.submitContext(getFailedContextConfiguration());
  }
}
 

@Override
public void onNext(final AllocatedEvaluator eb) {
  LOG.log(Level.FINE, "AllocatedEvaluator: " + eb);
  try {
    eb.submitContext(ContextConfiguration.CONF
        .set(ContextConfiguration.IDENTIFIER, "EvaluatorReuse").build());
  } catch (final BindException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public void onNext(final AllocatedEvaluator allocatedEvaluator) {
  if (isFirstEvaluator.compareAndSet(true, false)) {
    allocatedEvaluator.submitContext(getFailedContextConfiguration());
  } else {
    allocatedEvaluator.submitContext(ContextConfiguration.CONF
        .set(ContextConfiguration.IDENTIFIER, ROOT_CONTEXT_ID)
        .build());
  }
}
 

@Override
public void onNext(final AllocatedEvaluator eval) {
  LOG.log(Level.INFO, "ENTER: FailDriverDelayedMsg.send(AllocatedEvaluator): {0}", eval);
  try {
    eval.submitContext(ContextConfiguration.CONF
        .set(ContextConfiguration.IDENTIFIER, "Context_" + eval.getId())
        .build());
  } catch (final BindException ex) {
    LOG.log(Level.WARNING, "Context configuration error", ex);
    throw new RuntimeException(ex);
  }
}
 

@Override
public Configuration getContextConfiguration(final AllocatedEvaluator allocatedEvaluator) {

  final NumberedSplit<InputSplit> numberedSplit =
      this.evaluatorToPartitionStrategy.getInputSplit(
          allocatedEvaluator.getEvaluatorDescriptor().getNodeDescriptor(),
          allocatedEvaluator.getId());

  return ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, DATA_LOAD_CONTEXT_PREFIX + numberedSplit.getIndex())
      .build();
}
 

@Override
public void onNext(final AllocatedEvaluator allocatedEvaluator) {
  final Configuration contextConfiguration = ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, "FailingEvaluator")
      .set(ContextConfiguration.ON_CONTEXT_STARTED, FailureSchedulingContextStartHandler.class)
      .build();
  allocatedEvaluator.submitContext(contextConfiguration);
}
 

@Override
public void submitTask(final Configuration taskConfiguration) {
  final Configuration contextConfiguration = ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, "RootContext_" + this.getId())
      .build();
  this.submitContextAndTask(contextConfiguration, taskConfiguration);
}
 

/**
 * Submit Task with configuration strings.
 * This method should be called from bridge and the configuration strings are
 * serialized at .Net side.
 * @param evaluatorConfiguration
 * @param taskConfiguration
 */
public void submitTask(final String evaluatorConfiguration, final String taskConfiguration) {
  final Configuration contextConfiguration = ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, "RootContext_" + this.getId())
      .build();
  final String contextConfigurationString = this.configurationSerializer.toString(contextConfiguration);
  this.launchWithConfigurationString(
      evaluatorConfiguration, contextConfigurationString,  Optional.<String>empty(), Optional.of(taskConfiguration));
}
 

@Override
public void onNext(final AllocatedEvaluator allocatedEvaluator) {
  LOG.log(Level.INFO, "Submitting Output Service to AllocatedEvaluator: {0}", allocatedEvaluator);
  final Configuration contextConfiguration = ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, "OutputServiceContext")
      .build();
  allocatedEvaluator.submitContextAndService(
      contextConfiguration, outputService.getServiceConfiguration());
}
 

@Override
public Configuration getContextConfiguration() {
  LOG.entering("GroupCommDriverImpl", "getContextConf");
  final Configuration retVal = ContextConfiguration.CONF.set(ContextConfiguration.IDENTIFIER,
      "GroupCommunicationContext-" + contextIds.getAndIncrement()).build();
  LOG.exiting("GroupCommDriverImpl", "getContextConf", confSerializer.toString(retVal));
  return retVal;
}
 

@Override
public void onNext(final AllocatedEvaluator allocatedEvaluator) {
  LOG.log(Level.INFO, "Submitting an id context to AllocatedEvaluator: {0}", allocatedEvaluator);
  final Configuration contextConfiguration = ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, "BroadcastContext-" +
          BroadcastDriver.this.numberOfAllocatedEvaluators.getAndDecrement())
      .build();
  allocatedEvaluator.submitContext(contextConfiguration);
}
 

@Override
public void onNext(final AllocatedEvaluator evaluator) {
  LOG.log(Level.INFO, "Evaluator is ready");
  synchronized (SchedulerDriver.this) {
    nActiveEval++;
    nRequestedEval--;
  }

  evaluator.submitContext(ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, "SchedulerContext")
      .build());
}
 

void submitContext(final AllocatedEvaluator evaluator, final String identifier) {
  evaluator.submitContext(ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, identifier)
      .build());
}
 

@VisibleForTesting
Configuration createContextConfig(int executorId) {
  return ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, executorId + "")
      .build();
}
 

@Override
public void onNext(final ActiveContext activeContext) {

  final String contextId = activeContext.getId();
  LOG.log(Level.FINER, "Context active: {0}", contextId);

  if (dataLoadingService.isDataLoadedContext(activeContext)) {

    final String lcContextId = "LineCountCtxt-" + ctrlCtxIds.getAndIncrement();
    LOG.log(Level.FINEST, "Submit LineCount context {0} to: {1}",
        new Object[]{lcContextId, contextId});

    final Configuration poisonedConfiguration = PoisonedConfiguration.CONTEXT_CONF
        .set(PoisonedConfiguration.CRASH_PROBABILITY, "0.4")
        .set(PoisonedConfiguration.CRASH_TIMEOUT, "1")
        .build();

    activeContext.submitContext(Tang.Factory.getTang()
        .newConfigurationBuilder(poisonedConfiguration,
            ContextConfiguration.CONF.set(ContextConfiguration.IDENTIFIER, lcContextId).build())
        .build());

  } else if (activeContext.getId().startsWith("LineCountCtxt")) {

    final String taskId = "LineCountTask-" + ctrlCtxIds.getAndIncrement();
    LOG.log(Level.FINEST, "Submit LineCount task {0} to: {1}", new Object[]{taskId, contextId});

    try {
      activeContext.submitTask(TaskConfiguration.CONF
          .set(TaskConfiguration.IDENTIFIER, taskId)
          .set(TaskConfiguration.TASK, LineCountingTask.class)
          .build());
    } catch (final BindException ex) {
      LOG.log(Level.SEVERE, "Configuration error in " + contextId, ex);
      throw new RuntimeException("Configuration error in " + contextId, ex);
    }
  } else {
    LOG.log(Level.FINEST, "Line count Compute Task {0} -- Closing", contextId);
    activeContext.close();
  }
}
 

@Override
public void onNext(final AllocatedEvaluator eval) {

  LOG.log(Level.INFO, "TIME: Allocated Evaluator {0}", eval.getId());

  final boolean runTask;
  final int nEval;
  final int nTask;

  synchronized (JobDriver.this) {
    runTask = numTasksStarted < numTasks;
    if (runTask) {
      ++numEvaluatorsStarted;
      if (isPiggyback) {
        ++numTasksStarted;
      }
    }
    nEval = numEvaluatorsStarted;
    nTask = numTasksStarted;
  }

  if (runTask) {

    final String contextId = String.format("Context_%06d", nEval);
    LOG.log(Level.INFO, "TIME: Submit Context {0} to Evaluator {1}",
        new Object[]{contextId, eval.getId()});

    try {

      final JavaConfigurationBuilder contextConfigBuilder =
          Tang.Factory.getTang().newConfigurationBuilder();

      contextConfigBuilder.addConfiguration(ContextConfiguration.CONF
          .set(ContextConfiguration.IDENTIFIER, contextId)
          .build());

      contextConfigBuilder.bindNamedParameter(Launch.Delay.class, delayStr);

      if (isPiggyback) {

        final String taskId = String.format("StartTask_%08d", nTask);
        final Configuration taskConfig = getTaskConfiguration(taskId);

        LOG.log(Level.INFO, "TIME: Submit Task {0} to Evaluator {1}",
            new Object[]{taskId, eval.getId()});

        eval.submitContextAndTask(contextConfigBuilder.build(), taskConfig);

      } else {
        eval.submitContext(contextConfigBuilder.build());
      }

    } catch (final BindException ex) {
      LOG.log(Level.SEVERE, "Failed to submit Context to Evaluator: " + eval.getId(), ex);
      throw new RuntimeException(ex);
    }
  } else {
    LOG.log(Level.INFO, "TIME: Close Evaluator {0}", eval.getId());
    eval.close();
  }
}
 

@Override
public void onNext(final AllocatedEvaluator eval) {

  try {

    taskId = failTaskName + "_" + eval.getId();
    LOG.log(Level.INFO, "Submit task: {0}", taskId);

    final Configuration contextConfig =
        ContextConfiguration.CONF.set(ContextConfiguration.IDENTIFIER, taskId).build();

    ConfigurationModule taskConfig =
        TaskConfiguration.CONF.set(TaskConfiguration.IDENTIFIER, taskId);

    switch (failTaskName) {
    case "FailTask":
      taskConfig = taskConfig.set(TaskConfiguration.TASK, FailTask.class);
      break;
    case "FailTaskCall":
      taskConfig = taskConfig.set(TaskConfiguration.TASK, FailTaskCall.class);
      break;
    case "FailTaskMsg":
      taskConfig = taskConfig
            .set(TaskConfiguration.TASK, FailTaskMsg.class)
            .set(TaskConfiguration.ON_MESSAGE, FailTaskMsg.class);
      break;
    case "FailTaskSuspend":
      taskConfig = taskConfig
            .set(TaskConfiguration.TASK, FailTaskSuspend.class)
            .set(TaskConfiguration.ON_SUSPEND, FailTaskSuspend.class);
      break;
    case "FailTaskStart":
      taskConfig = taskConfig
            .set(TaskConfiguration.TASK, FailTaskStart.class)
            .set(TaskConfiguration.ON_TASK_STARTED, FailTaskStart.class);
      break;
    case "FailTaskStop":
      taskConfig = taskConfig
            .set(TaskConfiguration.TASK, FailTaskStop.class)
            .set(TaskConfiguration.ON_TASK_STOP, FailTaskStop.class)
            .set(TaskConfiguration.ON_CLOSE, FailTaskStop.CloseEventHandler.class);
      break;
    case "FailTaskClose":
      taskConfig = taskConfig
            .set(TaskConfiguration.TASK, FailTaskClose.class)
            .set(TaskConfiguration.ON_CLOSE, FailTaskClose.class);
      break;
    default:
      break;
    }

    eval.submitContextAndTask(contextConfig, taskConfig.build());

  } catch (final BindException ex) {
    LOG.log(Level.WARNING, "Configuration error", ex);
    throw new DriverSideFailure("Configuration error", ex);
  }
}
 

private Configuration getFailedContextConfiguration() {
  return ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, "FAILED_CONTEXT")
      .set(ContextConfiguration.ON_CONTEXT_STARTED, FailedContextHandler.class)
      .build();
}