Java Code Examples for org.apache.reef.driver.evaluator.AllocatedEvaluator#submitContext()

@Override
public void onNext(final AllocatedEvaluator allocatedEvaluator) {

  LOG.log(Level.FINE, "Submitting root context");

  try {

    final Configuration contextConfiguration = ContextConfiguration.CONF
        .set(ContextConfiguration.ON_CONTEXT_STARTED, ContextStartHandler1.class)
        .set(ContextConfiguration.ON_CONTEXT_STOP, ContextStopHandler1.class)
        .set(ContextConfiguration.IDENTIFIER, CONTEXT_1_IDENTIFIER)
        .build();

    allocatedEvaluator.submitContext(contextConfiguration);

    synchronized (SubContextDriver.this) {
      SubContextDriver.this.state = State.CONTEXT_1_SUBMITTED;
    }

  } catch (final BindException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public void onNext(final AllocatedEvaluator eval) {
  try {

    LOG.log(Level.INFO, "Allocated Evaluator: {0}", eval.getId());

    final Configuration thisContextConfiguration = ContextConfiguration.CONF.set(
        ContextConfiguration.IDENTIFIER, eval.getId() + "_context").build();

    eval.submitContext(Tang.Factory.getTang()
        .newConfigurationBuilder(thisContextConfiguration, contextConfig).build());

  } catch (final BindException ex) {
    throw new RuntimeException(ex);
  }
}
 

/**
 * Launches executor once a container is allocated.
 * @param resourceSpecification of the executor to be launched.
 * @param executorId of the executor to be launched.
 * @param allocatedContainer the allocated container.
 * @param executorConfiguration executor related configuration.
 */
private void onContainerAllocated(final ResourceSpecification resourceSpecification,
                                  final String executorId,
                                  final AllocatedEvaluator allocatedContainer,
                                  final Configuration executorConfiguration) {
  LOG.info("Container type (" + resourceSpecification.getContainerType()
      + ") allocated, will be used for [" + executorId + "]");
  pendingContextIdToResourceSpec.put(executorId, resourceSpecification);

  allocatedContainer.submitContext(executorConfiguration);
}
 

@Override
public void onNext(AllocatedEvaluator evaluator) {
  EvaluatorDescriptor descriptor = evaluator.getEvaluatorDescriptor();
  LOG.log(Level.INFO, String.format("New container received, id: %s, mem: %d, cores: %d",
      evaluator.getId(), descriptor.getMemory(), descriptor.getNumberOfCores()));

  Optional<HeronWorker> result;
  HeronWorker worker;
  synchronized (containerPlans) {
    Set<HeronWorker> workersAwaitingAllocation = getWorkersAwaitingAllocation();

    if (workersAwaitingAllocation.isEmpty()) {
      LOG.log(Level.INFO, "Could not find any workers waiting for allocation, closing {0}",
          evaluator.getId());
      evaluator.close();
      return;
    }

    result = findLargestFittingWorker(evaluator, workersAwaitingAllocation, true);
    if (!result.isPresent()) {
      LOG.warning("Could not find a fitting worker in awaiting workers");
      // TODO may need counting of missed allocation
      evaluator.close();
      return;
    }

    worker = result.get();
    LOG.info(String.format("Worker:%d, cores:%d, mem:%s fits in the allocated container",
        worker.workerId, worker.cores, worker.mem));
    workersAwaitingAllocation.remove(worker);
    multiKeyWorkerMap.assignEvaluatorToWorker(worker, evaluator);
  }

  LOG.log(Level.INFO, "Activating container {0} for heron worker, id: {1}",
      new Object[]{evaluator.getId(), worker.workerId});
  Configuration context = createContextConfig(worker.workerId);
  evaluator.submitContext(context);
}
 

@Override
public void onNext(final AllocatedEvaluator allocatedEvaluator) {
  if (isFirstEvaluator.compareAndSet(true, false)) {
    allocatedEvaluator.submitContext(getFailedContextConfiguration());
  } else {
    allocatedEvaluator.submitContext(ContextConfiguration.CONF
        .set(ContextConfiguration.IDENTIFIER, ROOT_CONTEXT_ID)
        .build());
  }
}
 

@Override
public void onNext(final AllocatedEvaluator eb) {
  LOG.log(Level.FINE, "AllocatedEvaluator: " + eb);
  try {
    eb.submitContext(ContextConfiguration.CONF
        .set(ContextConfiguration.IDENTIFIER, "EvaluatorReuse").build());
  } catch (final BindException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public void onNext(final AllocatedEvaluator eval) {
  LOG.log(Level.INFO, "ENTER: FailDriverDelayedMsg.send(AllocatedEvaluator): {0}", eval);
  try {
    eval.submitContext(ContextConfiguration.CONF
        .set(ContextConfiguration.IDENTIFIER, "Context_" + eval.getId())
        .build());
  } catch (final BindException ex) {
    LOG.log(Level.WARNING, "Context configuration error", ex);
    throw new RuntimeException(ex);
  }
}
 

@Override
public void onNext(final AllocatedEvaluator allocatedEvaluator) {
  final Configuration contextConfiguration = ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, "FailingEvaluator")
      .set(ContextConfiguration.ON_CONTEXT_STARTED, FailureSchedulingContextStartHandler.class)
      .build();
  allocatedEvaluator.submitContext(contextConfiguration);
}
 

@Override
public void onNext(final AllocatedEvaluator allocatedEvaluator) {
  LOG.log(Level.INFO, "Submitting an id context to AllocatedEvaluator: {0}", allocatedEvaluator);
  final Configuration contextConfiguration = ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, "BroadcastContext-" +
          BroadcastDriver.this.numberOfAllocatedEvaluators.getAndDecrement())
      .build();
  allocatedEvaluator.submitContext(contextConfiguration);
}
 

@Override
public void onNext(final AllocatedEvaluator evaluator) {
  LOG.log(Level.INFO, "Evaluator is ready");
  synchronized (SchedulerDriver.this) {
    nActiveEval++;
    nRequestedEval--;
  }

  evaluator.submitContext(ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, "SchedulerContext")
      .build());
}
 

/**
 * Take the necessary actions in container manager once a container a is allocated.
 *
 * @param executorId            of the executor to launch on this container.
 * @param allocatedContainer    the allocated container.
 * @param executorConfiguration executor related configuration.
 */
public void onContainerAllocated(final String executorId,
                                 final AllocatedEvaluator allocatedContainer,
                                 final Configuration executorConfiguration) {
  if (isTerminated) {
    LOG.info("ContainerManager is terminated, closing {}", allocatedContainer.getId());
    allocatedContainer.close();
    return;
  }

  final ResourceSpecification resourceSpecification = selectResourceSpecForContainer();
  final List<Configuration> configurationsToMerge = new ArrayList<>();

  evaluatorIdToResourceSpec.put(allocatedContainer.getId(), resourceSpecification);

  LOG.info("Container type (" + resourceSpecification.getContainerType()
    + ") allocated, will be used for [" + executorId + "]");
  pendingContextIdToResourceSpec.put(executorId, resourceSpecification);

  configurationsToMerge.add(executorConfiguration);

  // ExecutorMemory handling
  configurationsToMerge.add(Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(JobConf.ExecutorMemoryMb.class, String.valueOf(resourceSpecification.getMemory()))
      .build()
  );

  // MaxOffheapRatio handling
  resourceSpecification.getMaxOffheapRatio().ifPresent(value ->
    configurationsToMerge.add(Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(JobConf.MaxOffheapRatio.class, String.valueOf(value))
      .build()
    )
  );

  // Poison handling
  resourceSpecification.getPoisonSec().ifPresent(value ->
    configurationsToMerge.add(Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(JobConf.ExecutorPoisonSec.class, String.valueOf(value))
      .build()
    )
  );

  allocatedContainer.submitContext(Configurations.merge(configurationsToMerge));
}
 

@Override
public void onNext(final AllocatedEvaluator eval) {

  LOG.log(Level.INFO, "TIME: Allocated Evaluator {0}", eval.getId());

  final boolean runTask;
  final int nEval;
  final int nTask;

  synchronized (JobDriver.this) {
    runTask = numTasksStarted < numTasks;
    if (runTask) {
      ++numEvaluatorsStarted;
      if (isPiggyback) {
        ++numTasksStarted;
      }
    }
    nEval = numEvaluatorsStarted;
    nTask = numTasksStarted;
  }

  if (runTask) {

    final String contextId = String.format("Context_%06d", nEval);
    LOG.log(Level.INFO, "TIME: Submit Context {0} to Evaluator {1}",
        new Object[]{contextId, eval.getId()});

    try {

      final JavaConfigurationBuilder contextConfigBuilder =
          Tang.Factory.getTang().newConfigurationBuilder();

      contextConfigBuilder.addConfiguration(ContextConfiguration.CONF
          .set(ContextConfiguration.IDENTIFIER, contextId)
          .build());

      contextConfigBuilder.bindNamedParameter(Launch.Delay.class, delayStr);

      if (isPiggyback) {

        final String taskId = String.format("StartTask_%08d", nTask);
        final Configuration taskConfig = getTaskConfiguration(taskId);

        LOG.log(Level.INFO, "TIME: Submit Task {0} to Evaluator {1}",
            new Object[]{taskId, eval.getId()});

        eval.submitContextAndTask(contextConfigBuilder.build(), taskConfig);

      } else {
        eval.submitContext(contextConfigBuilder.build());
      }

    } catch (final BindException ex) {
      LOG.log(Level.SEVERE, "Failed to submit Context to Evaluator: " + eval.getId(), ex);
      throw new RuntimeException(ex);
    }
  } else {
    LOG.log(Level.INFO, "TIME: Close Evaluator {0}", eval.getId());
    eval.close();
  }
}
 

void submitContext(final AllocatedEvaluator evaluator, final String identifier) {
  evaluator.submitContext(ContextConfiguration.CONF
      .set(ContextConfiguration.IDENTIFIER, identifier)
      .build());
}