org.apache.reef.driver.context.ActiveContext Java Examples

/**
 * @return a FailedContext for the case of an EvaluatorFailure.
 */
public synchronized FailedContext getFailedContextForEvaluatorFailure() {

  final String id = this.getId();
  final Optional<String> description = Optional.empty();
  final Optional<byte[]> data = Optional.empty();
  final Optional<Throwable> cause = Optional.empty();
  final String message = "Evaluator Failure";

  final Optional<ActiveContext> parentContext = getParentId().isPresent() ?
      Optional.<ActiveContext>of(this.contextRepresenters.getContext(getParentId().get())) :
      Optional.<ActiveContext>empty();

  final String evaluatorID = getEvaluatorId();

  return new FailedContextImpl(
      id, message, description, cause, data, parentContext, this.evaluatorDescriptor, evaluatorID);
}
 

@Override
public void fail(final ActiveContext context) {
  if (this.clock.get().isClosed()) {
    throw new IllegalStateException("clock is closed");
  }

  final MockAllocatedEvaluator evaluator = ((MockActiveContext) context).getEvaluator();

  // Root context failure shuts evaluator down.
  if (!((MockActiveContext) context).getParentContext().isPresent()) {
    allocatedEvaluatorMap.remove(evaluator.getId());
    post(this.completedEvaluatorHandlers, new CompletedEvaluator() {
      @Override
      public String getId() {
        return evaluator.getId();
      }
    });
  }

  this.allocatedContextsMap.get(evaluator.getId()).remove(context);
  post(this.contextFailedHandlers, new MockFailedContext((MockActiveContext) context));
}
 

@Override
public void failedTaskHandler(final TaskInfo request, final StreamObserver<Void> responseObserver) {
  try (ObserverCleanup cleanup = ObserverCleanup.of(responseObserver)) {
    LOG.log(Level.INFO, "Failed task id {0}", request.getTaskId());
    ActiveContextBridge context = request.hasContext() ?
        addContextIfMissing(request.getContext()) : null;
    this.clientDriverDispatcher.get().dispatch(
        new FailedTask(
            request.getTaskId(),
            request.getException().getMessage(),
            Optional.of(request.getException().getName()),
            request.getException().getData().isEmpty() ?
                Optional.<Throwable>of(new EvaluatorException(request.getException().getMessage())) :
                this.exceptionCodec.fromBytes(request.getException().getData().toByteArray()),
            Optional.<byte[]>empty(),
            Optional.<ActiveContext>ofNullable(context)));
  }
}
 

@Override
public void runningTaskHandler(final RunningTask task) {
  synchronized (this) {
    if (this.clientStub != null) {
      final ActiveContext context = task.getActiveContext();
      if (!this.activeContextMap.containsKey(context.getId())) {
        this.activeContextMap.put(context.getId(), context);
      }
      this.runningTaskMap.put(task.getId(), task);
      this.clientStub.runningTaskHandler(
          TaskInfo.newBuilder()
              .setTaskId(task.getId())
              .setContext(GRPCUtils.toContextInfo(context))
              .build());
    } else {
      LOG.log(Level.WARNING, "client shutdown has already completed");
    }
  }
}
 

private void submitTask(final ActiveContext activeContext) {

      assert groupCommDriver.isConfigured(activeContext);

      final Configuration partialTaskConfiguration;
      if (activeContext.getId().equals(communicationsGroupMasterContextId) && !masterTaskSubmitted()) {
        partialTaskConfiguration = getMasterTaskConfiguration();
        LOG.info("Submitting MasterTask conf");
      } else {
        partialTaskConfiguration = getSlaveTaskConfiguration(getSlaveId(activeContext));
        // partialTaskConfiguration = Configurations.merge(
        //     getSlaveTaskConfiguration(getSlaveId(activeContext)),
        //     getTaskPoisonConfiguration());
        LOG.info("Submitting SlaveTask conf");
      }
      communicationsGroup.addTask(partialTaskConfiguration);
      final Configuration taskConfiguration = groupCommDriver.getTaskConfiguration(partialTaskConfiguration);
      LOG.log(Level.FINEST, "{0}", confSerializer.toString(taskConfiguration));
      activeContext.submitTask(taskConfiguration);
    }
 

/**
 * Called when an executor is launched on a container for this runtime.
 * @param activeContext of the launched executor.
 * @return true if all requested executors have been launched, false otherwise.
 */
public boolean onExecutorLaunched(final ActiveContext activeContext) {
  final Callable<Boolean> processExecutorLaunchedEvent = () -> {
    containerManager.onExecutorLaunched(activeContext);
    scheduler.onExecutorAdded(activeContext.getId());
    return (resourceRequestCount.decrementAndGet() == 0);
  };

  final boolean eventResult;
  try {
    eventResult = masterControlEventExecutor.submit(processExecutorLaunchedEvent).get();
  } catch (final Exception e) {
    throw new ContainerException(e);
  }
  return eventResult;
}
 

/**
 * Submit a Task that execute the command to a single Evaluator.
 * This method is called from <code>submitTask(cmd)</code>.
 */
private void submit(final ActiveContext context, final String command) {
  try {
    LOG.log(Level.INFO, "Send command {0} to context: {1}", new Object[]{command, context});
    final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder();
    cb.addConfiguration(
        TaskConfiguration.CONF
            .set(TaskConfiguration.IDENTIFIER, context.getId() + "_task")
            .set(TaskConfiguration.TASK, ShellTask.class)
            .build()
    );
    cb.bindNamedParameter(Command.class, command);
    context.submitTask(cb.build());
  } catch (final BindException ex) {
    LOG.log(Level.SEVERE, "Bad Task configuration for context: " + context.getId(), ex);
    context.close();
    throw new RuntimeException(ex);
  }
}
 

@Test
public void activeContextStateHandlerTest() throws InjectionException {

  final ReefEventStateManager.ActiveContextStateHandler h =
      this.injector.getInstance(ReefEventStateManager.ActiveContextStateHandler.class);
  Assert.assertNotNull(h);

  final MockActiveContext activityContext = injector.getInstance(MockActiveContext.class);
  h.onNext(activityContext);

  final ReefEventStateManager reefEventStateManager =
      this.injector.getInstance(ReefEventStateManager.class);

  final Map<String, ActiveContext> contexts = reefEventStateManager.getContexts();
  Assert.assertEquals(1, contexts.size());

  for (final ActiveContext context : contexts.values()) {
    Assert.assertEquals(activityContext.getId(), context.getId());
  }
}
 

@Override
public void fail(final AllocatedEvaluator evaluator) {
  if (this.clock.get().isClosed()) {
    throw new IllegalStateException("clock is closed");
  }
  if (this.allocatedEvaluatorMap.remove(evaluator.getId()) == null) {
    throw new IllegalStateException("unknown evaluator " + evaluator);
  }
  FailedTask failedTask = null;
  if (this.runningTasks.containsKey(evaluator.getId())) {
    final RunningTask task = this.runningTasks.remove(evaluator.getId());
    failedTask = new FailedTask(
        task.getId(),
        "mock",
        Optional.<String>empty(),
        Optional.<Throwable>empty(),
        Optional.<byte[]>empty(),
        Optional.<ActiveContext>of(task.getActiveContext()));
  }
  final List<FailedContext> failedContexts = new ArrayList<>();
  for (final MockActiveContext context : this.allocatedContextsMap.get(evaluator.getId())) {
    failedContexts.add(new MockFailedContext(context));
  }
  post(this.failedEvaluatorHandlers, new MockFailedEvaluator(
      evaluator.getId(), failedContexts, Optional.ofNullable(failedTask)));
}
 

/**
 * Simulate the launch of executors.
 */
private void setUpExecutors() {
  final List<Pair<Integer, ResourceSpecification>> resourceSpecs =
    Util.parseResourceSpecificationString(resourceSpecificationString);
  // Role of ActiveContextHandler + RuntimeMaster.onExecuterLaunched.
  final AtomicInteger executorIdGenerator = new AtomicInteger(0);
  final AtomicInteger resourceRequestCount = new AtomicInteger(0);
  resourceSpecs.forEach(p -> {
    for (int i = 0; i < p.left(); i++) {
      resourceRequestCount.getAndIncrement();
      final ActiveContext ac = new SimulationEvaluatorActiveContext(executorIdGenerator.getAndIncrement());
      this.onExecutorAdded(new DefaultExecutorRepresenter(ac.getId(), p.right(),
        new SimulationMessageSender(ac.getId(), this), ac, serializationExecutorService, ac.getId()));
    }
  });
  this.metricCountDownLatch = new CountDownLatch(resourceRequestCount.get());
}
 

/**
 * Called when an executor is launched on a container for this runtime.
 *
 * @param activeContext of the launched executor.
 * @return true if all requested executors have been launched, false otherwise.
 */
public boolean onExecutorLaunched(final ActiveContext activeContext) {
  final Callable<Boolean> processExecutorLaunchedEvent = () -> {
    final Optional<ExecutorRepresenter> executor = containerManager.onContainerLaunched(activeContext);
    if (executor.isPresent()) {
      scheduler.onExecutorAdded(executor.get());
      return (resourceRequestCount.decrementAndGet() == 0);
    } else {
      return false;
    }
  };

  final boolean eventResult;
  try {
    eventResult = runtimeMasterThread.submit(processExecutorLaunchedEvent).get();
  } catch (final Exception e) {
    throw new ContainerException(e);
  }
  return eventResult;
}
 

MockRunningTask(
    final MockRuntimeDriver mockRuntimeDriver,
    final String taskID,
    final ActiveContext context) {
  this.mockRuntimeDriver = mockRuntimeDriver;
  this.taskID = taskID;
  this.context = context;
}
 

@Override
public Collection<ActiveContext> getCurrentActiveContexts() {
  if (this.clock.get().isClosed()) {
    throw new IllegalStateException("clock is closed");
  }
  final List<ActiveContext> currentActiveContexts = new ArrayList<>();
  for (final List<MockActiveContext> contexts : this.allocatedContextsMap.values()) {
    currentActiveContexts.addAll(contexts);
  }
  return currentActiveContexts;
}
 

@Override
public void onNext(final ActiveContext activeContext) {
  LOG.log(Level.INFO, "Got active context: {0}", activeContext.getId());
  if (jobRunning(activeContext)) {
    if (!groupCommDriver.isConfigured(activeContext)) {
      // The Context is not configured with the group communications service let's do that.
      submitGroupCommunicationsService(activeContext);
    } else {
      // The group communications service is already active on this context. We can submit the task.
      submitTask(activeContext);
    }
  }
}
 

private ActiveContext createMockContext(final String id,
                                        final EvaluatorDescriptor descriptor) {
  final ActiveContext mockedContext = mock(ActiveContext.class);
  when(mockedContext.getId()).thenReturn(id);
  when(mockedContext.getEvaluatorDescriptor()).thenReturn(descriptor);
  return mockedContext;
}
 

public FailedContextBridge(final FailedContext failedContext, final ActiveContextBridgeFactory factory) {
  jfailedContext = failedContext;
  evaluatorDescriptor = failedContext.getEvaluatorDescriptor();
  evaluatorId = failedContext.getEvaluatorId();
  contextId = failedContext.getId();
  if (failedContext.getParentContext().isPresent()) {
    final ActiveContext parent = failedContext.getParentContext().get();
    this.parentContextId = parent.getId();
    this.parentContext = factory.getActiveContextBridge(parent);
  } else {
    this.parentContextId = null;
    this.parentContext = null;
  }

}
 

@Override
public void onNext(final ClosedContext closedContext) {
  LOG.log(Level.FINE, "Got closed context: {0}", closedContext.getId());
  final ActiveContext parentContext = closedContext.getParentContext();
  if (parentContext != null) {
    LOG.log(Level.FINE, "Closing parent context: {0}", parentContext.getId());
    parentContext.close();
  }
}
 

@Override
public void onNext(final CompletedTask task) {

  final ActiveContext context = task.getActiveContext();
  LOG.log(Level.INFO, "TIME: Completed Task {0} on Evaluator {1}",
      new Object[]{task.getId(), context.getEvaluatorId()});

  final boolean runTask;
  final int nTask;
  synchronized (JobDriver.this) {
    runTask = numTasksStarted < numTasks;
    if (runTask) {
      ++numTasksStarted;
    }
    nTask = numTasksStarted;
  }

  if (runTask) {
    final String taskId = String.format("Task_%08d", nTask);
    LOG.log(Level.INFO, "TIME: Submit Task {0} to Evaluator {1}",
        new Object[]{taskId, context.getEvaluatorId()});
    context.submitTask(getTaskConfiguration(taskId));
  } else {
    LOG.log(Level.INFO, "TIME: Close Evaluator {0}", context.getEvaluatorId());
    context.close();
  }
}
 

@Override
public void onNext(final ActiveContext context) {
  if (!evaluatorId2ContextId2ContextMap.containsKey(context.getEvaluatorId())) {
    evaluatorId2ContextId2ContextMap.put(context.getEvaluatorId(), new HashMap<String, ActiveContext>());
  }
  if (evaluatorId2ContextId2ContextMap.get(context.getEvaluatorId()).containsKey(context.getId())) {
    throw new IllegalStateException(
            String.format("Context %s on evaluator %s already exists on evaluator with " +
                    "same identifier", context.getId(), context.getEvaluatorId()));
  }
  evaluatorId2ContextId2ContextMap.get(context.getEvaluatorId()).put(context.getId(), context);
}
 

@Override
public void onNext(final ActiveContext context) {

  LOG.log(Level.INFO, "TIME: Active Context {0}", context.getId());

  if (isPiggyback) {
    return; // Task already submitted
  }

  final boolean runTask;
  final int nTask;

  synchronized (JobDriver.this) {
    runTask = numTasksStarted < numTasks;
    if (runTask) {
      ++numTasksStarted;
    }
    nTask = numTasksStarted;
  }

  if (runTask) {
    final String taskId = String.format("StartTask_%08d", nTask);
    LOG.log(Level.INFO, "TIME: Submit Task {0} to Evaluator {1}",
        new Object[]{taskId, context.getEvaluatorId()});
    context.submitTask(getTaskConfiguration(taskId));
  } else {
    context.close();
  }
}
 

public ClosedContextBridge(
    final String contextId,
    final String evaluatorId,
    final ActiveContext parentContext,
    final EvaluatorDescriptor evaluatorDescriptor) {
  this.contextId = contextId;
  this.evaluatorId = evaluatorId;
  this.parentContext = parentContext;
  this.evaluatorDescriptor = evaluatorDescriptor;
}
 

@Override
public void onNext(final ActiveContext activeContext) {
  final int count = contextCounter.getAndIncrement();

  if (count <= 1) {
    LOG.log(Level.INFO, "{0} will be handled after tasks in Group1 started", activeContext);
    activeContextsToBeHandled.add(activeContext);
  } else {
    // Add task to Group1
    submitTask(activeContext, 0);
  }
}
 

@Override
public void onNext(final RunningTask runningTask) {
  LOG.log(Level.INFO, "{0} has started", runningTask);
  final int count = runningTaskCounter.getAndIncrement();
  // After two tasks has started, submit tasks to the active contexts in activeContextsToBeHandled
  if (count == 1) {
    for (final ActiveContext activeContext : activeContextsToBeHandled) {
      // Add task to Group2
      submitTask(activeContext, 1);
    }
  }
}
 

@Override
public void onNext(final ActiveContext activeContext) {
  final Configuration paramConf = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(GroupCommServiceInjectionParameter.class, Integer.toString(OFFSET))
      .build();

  if (masterTaskSubmitted.compareAndSet(false, true)) {
    final Configuration masterTaskPartialConf = TaskConfiguration.CONF
        .set(TaskConfiguration.IDENTIFIER, GroupCommServiceInjectionMasterTask.TASK_ID)
        .set(TaskConfiguration.TASK, GroupCommServiceInjectionMasterTask.class)
        .build();
    commGroupDriver.addTask(masterTaskPartialConf);

    final Configuration masterTaskFinalConf = groupCommDriver.getTaskConfiguration(
        Configurations.merge(paramConf, masterTaskPartialConf));
    activeContext.submitTask(masterTaskFinalConf);

  } else {
    final Configuration slaveTaskPartialConf = TaskConfiguration.CONF
        .set(TaskConfiguration.IDENTIFIER, GroupCommServiceInjectionSlaveTask.TASK_ID)
        .set(TaskConfiguration.TASK, GroupCommServiceInjectionSlaveTask.class)
        .build();
    commGroupDriver.addTask(slaveTaskPartialConf);

    final Configuration slaveTaskFinalConf = groupCommDriver.getTaskConfiguration(
        Configurations.merge(paramConf, slaveTaskPartialConf));
    activeContext.submitTask(slaveTaskFinalConf);
  }
}
 

private void nextPass(final ActiveContext activeContext) {
  try {
    activeContext.submitTask(TaskConfiguration.CONF
        .set(TaskConfiguration.IDENTIFIER, "StatePassing-" + pass)
        .set(TaskConfiguration.TASK, StatePassingTask.class)
        .build());
    ++pass;
  } catch (final BindException e) {
    throw new RuntimeException(e);
  }
}
 

/**
 * @param parentContext       the parent context.
 * @param contextID           the id of the closed context
 * @param evaluatorId         the id of the evaluator on which the context was closed
 * @param evaluatorDescriptor the descriptor of the evaluator on which the context was closed.
 */
public ClosedContextImpl(final ActiveContext parentContext,
                         final String contextID,
                         final String evaluatorId,
                         final EvaluatorDescriptor evaluatorDescriptor) {
  this.parentContext = parentContext;
  this.contextID = contextID;
  this.evaluatorId = evaluatorId;
  this.evaluatorDescriptor = evaluatorDescriptor;
}
 

/**
 * @param id                  Identifier of the entity that produced the error.
 * @param message             One-line error message.
 * @param description         Long error description.
 * @param cause               Java Exception that caused the error.
 * @param data                byte array that contains serialized version of the error.
 * @param parentContext       the parent context, if there is one.
 * @param evaluatorDescriptor the descriptor of the Evaluator this context failed on.
 * @param evaluatorID         the id of the Evaluator this context failed on.
 */
public FailedContextImpl(final String id,
                         final String message,
                         final Optional<String> description,
                         final Optional<Throwable> cause,
                         final Optional<byte[]> data,
                         final Optional<ActiveContext> parentContext,
                         final EvaluatorDescriptor evaluatorDescriptor,
                         final String evaluatorID) {
  super(id, message, description, cause, data);
  this.parentContext = parentContext;
  this.evaluatorDescriptor = evaluatorDescriptor;
  this.evaluatorID = evaluatorID;
}
 

@Override
public synchronized void onNext(final ActiveContext context) {
  LOG.log(Level.INFO, "Active Context: {0}", context.getId());
  try {
    context.submitTask(TaskConfiguration.CONF
        .set(TaskConfiguration.IDENTIFIER, context.getId() + "_task")
        .set(TaskConfiguration.TASK, SuspendTestTask.class)
        .set(TaskConfiguration.ON_SUSPEND, SuspendTestTask.SuspendHandler.class)
        .set(TaskConfiguration.ON_SEND_MESSAGE, SuspendTestTask.class)
        .build());
  } catch (final BindException ex) {
    LOG.log(Level.SEVERE, "Bad Task configuration for context: " + context.getId(), ex);
    throw new RuntimeException(ex);
  }
}
 

public RunningTaskBridge(
    final DriverServiceClient driverServiceClient,
    final String taskId,
    final ActiveContext context) {
  this.driverServiceClient = driverServiceClient;
  this.taskId = taskId;
  this.context = context;
}
 

/**
 * Pick up a command from the queue and run it. Wait until
 * any command coming up if no command exists.
 */
private synchronized void waitForCommands(final ActiveContext context) {
  while (!scheduler.hasPendingTasks()) {
    // Wait until any command enters in the queue
    try {
      wait();
    } catch (final InterruptedException e) {
      LOG.log(Level.WARNING, "InterruptedException occurred in SchedulerDriver", e);
    }
  }
  // When wakes up, run the first command from the queue.
  state = State.RUNNING;
  scheduler.submitTask(context);
}