org.apache.reef.driver.evaluator.EvaluatorRequestor Java Examples

@Before
public void createMocks() throws IOException {
  mockRequestor = mock(EvaluatorRequestor.class);
  driver = new HeronMasterDriver(mockRequestor,
      new REEFFileNames(),
      "yarn",
      "heron",
      "testTopology",
      "env",
      "jar",
      "package",
      "core",
      0,
      false);
  spyDriver = spy(driver);
  doReturn("").when(spyDriver).getComponentRamMap();
}
 

@Inject
private GroupCommServiceInjectionDriver(final EvaluatorRequestor evaluatorRequestor,
                                        final GroupCommDriver groupCommDriver) {
  this.evaluatorRequestor = evaluatorRequestor;
  this.groupCommDriver = groupCommDriver;
  this.commGroupDriver =
      groupCommDriver.newCommunicationGroup(GroupCommServiceInjectionGroupName.class, 2);

  this.commGroupDriver
      .addBroadcast(GroupCommServiceInjectionBroadcast.class,
          BroadcastOperatorSpec.newBuilder()
              .setSenderId(GroupCommServiceInjectionMasterTask.TASK_ID)
              .setDataCodecClass(GroupCommServiceInjectionCodec.class)
              .build())
      .finalise();
}
 

/**
 * Job driver constructor - instantiated via TANG.
 *
 * @param requestor evaluator requestor object used to create new evaluator containers.
 */
@Inject
private HelloJVMOptionsDriver(final EvaluatorRequestor requestor,
                              final JVMProcessFactory jvmProcessFactory) {
  this.requestor = requestor;
  this.jvmProcessFactory = jvmProcessFactory;
  LOG.log(Level.FINE, "Instantiated 'HelloDriver'");
}
 

/**
 * If only memory, no count is given, 1 evaluator should be requested.
 */
@Test
public void testMemoryOnly() {
  final int memory = 777;
  final DummyRequestHandler requestHandler = new DummyRequestHandler();
  final EvaluatorRequestor evaluatorRequestor =
      new EvaluatorRequestorImpl(resourceCatalog, requestHandler, loggingScopeFactory);
  evaluatorRequestor.submit(EvaluatorRequest.newBuilder().setMemory(memory).build());
  Assert.assertEquals("Memory request did not make it",
      memory, requestHandler.get().getMemorySize().get().intValue());
  Assert.assertEquals("Number of requests did not make it", 1, requestHandler.get().getResourceCount());
}
 

/**
 * Checks whether memory and count make it correctly.
 */
@Test
public void testMemoryAndCount() {
  final int memory = 777;
  final int count = 9;
  final DummyRequestHandler requestHandler = new DummyRequestHandler();
  final EvaluatorRequestor evaluatorRequestor =
      new EvaluatorRequestorImpl(resourceCatalog, requestHandler, loggingScopeFactory);
  evaluatorRequestor.submit(EvaluatorRequest.newBuilder().setMemory(memory).setNumber(count).build());
  Assert.assertEquals("Memory request did not make it",
      memory, requestHandler.get().getMemorySize().get().intValue());
  Assert.assertEquals("Number of requests did not make it", count, requestHandler.get().getResourceCount());
}
 

/**
 * Expect an IllegalArgumentException when a non-positive memory amount is passed.
 */
@Test(expected = IllegalArgumentException.class)
public void testIllegalMemory() {
  final int memory = 0;
  final int count = 1;
  final DummyRequestHandler requestHandler = new DummyRequestHandler();
  final EvaluatorRequestor evaluatorRequestor =
      new EvaluatorRequestorImpl(resourceCatalog, requestHandler, loggingScopeFactory);
  evaluatorRequestor.submit(EvaluatorRequest.newBuilder().setMemory(memory).setNumberOfCores(1).setNumber(count)
      .build());
}
 

/**
 * Expect an IllegalArgumentException when a non-positive evaluator count is passed.
 */
@Test(expected = IllegalArgumentException.class)
public void testIllegalCount() {
  final int memory = 128;
  final int count = 0;
  final DummyRequestHandler requestHandler = new DummyRequestHandler();
  final EvaluatorRequestor evaluatorRequestor =
      new EvaluatorRequestorImpl(resourceCatalog, requestHandler, loggingScopeFactory);
  evaluatorRequestor.submit(EvaluatorRequest.newBuilder().setMemory(memory).setNumberOfCores(1).setNumber(count)
      .build());
}
 

/**
 * Job driver constructor.
 * All parameters are injected from TANG automatically.
 *
 * @param evaluatorRequestor is used to request Evaluators.
 */
@Inject
JobDriver(final EvaluatorRequestor evaluatorRequestor,
          @Parameter(Launch.Piggyback.class) final Boolean isPiggyback,
          @Parameter(Launch.NumEvaluators.class) final Integer numEvaluators,
          @Parameter(Launch.NumTasks.class) final Integer numTasks,
          @Parameter(Launch.Delay.class) final Integer delay) {
  this.evaluatorRequestor = evaluatorRequestor;
  this.isPiggyback = isPiggyback;
  this.numEvaluators = numEvaluators;
  this.numTasks = numTasks;
  this.delayStr = "" + delay;
}
 

@Inject
private ShellDriver(
    final EvaluatorRequestor requestor,
    @Parameter(NumEvaluators.class) final int numEvaluators,
    @Parameter(Command.class) final String command) {

  this.requestor = requestor;
  this.numEvaluators = numEvaluators;
  this.command = command;
}
 

/**
 * Job driver constructor - instantiated via TANG.
 *
 * @param requestor evaluator requestor object used to create new evaluator containers.
 * @param outputService output service object.
 */
@Inject
public OutputServiceDriver(final EvaluatorRequestor requestor,
                           final OutputService outputService) {
  LOG.log(Level.FINE, "Instantiated 'OutputServiceDriver'");
  this.requestor = requestor;
  this.outputService = outputService;
}
 

@Inject
private SchedulerDriver(final EvaluatorRequestor requestor,
                        @Parameter(SchedulerREEF.Retain.class) final boolean retainable,
                        final Scheduler scheduler) {
  this.requestor = requestor;
  this.scheduler = scheduler;
  this.retainable = retainable;
}
 

@Before
public void setUp() throws InjectionException {

  final MessageEnvironment mockMsgEnv = mock(MessageEnvironment.class);
  when(mockMsgEnv.asyncConnect(anyString(), anyString())).thenReturn(mock(Future.class));
  final Configuration configuration = Tang.Factory.getTang().newConfigurationBuilder()
    .bindNamedParameter(JobConf.ScheduleSerThread.class, "1")
    .build();
  final Injector injector = Tang.Factory.getTang().newInjector(configuration);
  injector.bindVolatileInstance(EvaluatorRequestor.class, mock(EvaluatorRequestor.class));
  injector.bindVolatileInstance(MessageEnvironment.class, mockMsgEnv);
  containerManager = injector.getInstance(ContainerManager.class);
}
 

/**
 * Job driver constructor.
 * All parameters are injected from TANG automatically.
 *
 * @param evaluatorRequestor is used to request Evaluators.
 * @param numCycles          number of cycles to run in the task.
 * @param delay              delay in seconds between cycles in the task.
 */
@Inject
SuspendDriver(
    final JobMessageObserver jobMessageObserver,
    final EvaluatorRequestor evaluatorRequestor,
    @Parameter(Launch.Local.class) final boolean isLocal,
    @Parameter(Launch.NumCycles.class) final int numCycles,
    @Parameter(Launch.Delay.class) final int delay) {

  this.jobMessageObserver = jobMessageObserver;
  this.evaluatorRequestor = evaluatorRequestor;

  try {

    final Configuration checkpointServiceConfig = FSCheckPointServiceConfiguration.CONF
        .set(FSCheckPointServiceConfiguration.IS_LOCAL, Boolean.toString(isLocal))
        .set(FSCheckPointServiceConfiguration.PATH, "/tmp")
        .set(FSCheckPointServiceConfiguration.PREFIX, "reef-checkpoint-")
        .set(FSCheckPointServiceConfiguration.REPLICATION_FACTOR, "3")
        .build();

    final JavaConfigurationBuilder cb = Tang.Factory.getTang().newConfigurationBuilder()
        .bindNamedParameter(Launch.NumCycles.class, Integer.toString(numCycles))
        .bindNamedParameter(Launch.Delay.class, Integer.toString(delay));

    cb.addConfiguration(checkpointServiceConfig);
    this.contextConfig = cb.build();

  } catch (final BindException ex) {
    throw new RuntimeException(ex);
  }
}
 

/**
 * Job Driver Constructor.
 *
 * @param requestor
 * @param clientCallBackHandler
 */
@Inject
public HttpShellJobDriver(final EvaluatorRequestor requestor,
                          final HttpServerShellCmdHandler.ClientCallBackHandler clientCallBackHandler) {
  this.evaluatorRequestor = requestor;
  this.httpCallbackHandler = clientCallBackHandler;
  LOG.log(Level.FINE, "Instantiated 'HttpShellJobDriver'");
}
 

/**
 * Job driver constructor - instantiated via TANG.
 *
 * @param requestor evaluator requestor object used to create new evaluator containers.
 */
@Inject
public HelloDriver(final EvaluatorRequestor requestor,
                   final CLRProcessFactory clrProcessFactory) {
  this.requestor = requestor;
  this.clrProcessFactory = clrProcessFactory;
}
 

/**
 * Tests whether the amount of memory is transferred correctly.
 */
@Test
public void testDifferentMemory() throws InjectionException {

  final LoggingScopeFactory loggingScopeFactory =
      Tang.Factory.getTang().newInjector().getInstance(LoggingScopeFactory.class);

  final EvaluatorRequestor evaluatorRequestor =
      new EvaluatorRequestorImpl(resourceCatalog, resourceRequestHandler, loggingScopeFactory);

  final EvaluatorRequest requestOne = EvaluatorRequest.newBuilder()
      .setNumber(1)
      .setMemory(64)
      .setNumberOfCores(1)
      .build();

  final EvaluatorRequest requestTwo = EvaluatorRequest.newBuilder()
      .setNumber(1)
      .setMemory(128)
      .setNumberOfCores(2)
      .build();

  evaluatorRequestor.submit(requestOne);
  Assert.assertEquals("Request in REEF and YARN form should have the same amount of memory",
      requestOne.getMegaBytes(),
      containerRequestHandler.getRequests()[0].getCapability().getMemory());

  evaluatorRequestor.submit(requestTwo);
  Assert.assertEquals("Request in REEF and YARN form should have the same amount of memory",
      requestTwo.getMegaBytes(),
      containerRequestHandler.getRequests()[0].getCapability().getMemory());

  evaluatorRequestor.submit(requestOne);
  Assert.assertNotEquals("Second YARN request should have different amount of memory",
      requestTwo.getMegaBytes(),
      containerRequestHandler.getRequests()[0].getCapability().getMemory());
}
 

@Test
public void testEvaluatorCount() throws InjectionException {

  final LoggingScopeFactory loggingScopeFactory =
      Tang.Factory.getTang().newInjector().getInstance(LoggingScopeFactory.class);

  final EvaluatorRequestor evaluatorRequestor =
      new EvaluatorRequestorImpl(resourceCatalog, resourceRequestHandler, loggingScopeFactory);

  final EvaluatorRequest requestOne = EvaluatorRequest.newBuilder()
      .setNumber(1)
      .setMemory(64)
      .setNumberOfCores(1)
      .build();

  final EvaluatorRequest requestTwo = EvaluatorRequest.newBuilder()
      .setNumber(2)
      .setMemory(128)
      .setNumberOfCores(2)
      .build();

  evaluatorRequestor.submit(requestOne);
  Assert.assertEquals("Request in REEF and YARN form should have the same number of Evaluators",
      requestOne.getNumber(),
      containerRequestHandler.getRequests().length);

  evaluatorRequestor.submit(requestTwo);
  Assert.assertEquals("Request in REEF and YARN form should have the same number of Evaluators",
      requestTwo.getNumber(),
      containerRequestHandler.getRequests().length);

  evaluatorRequestor.submit(requestTwo);
  Assert.assertNotEquals("Second YARN request should have different number of Evaluators",
      requestOne.getNumber(),
      containerRequestHandler.getRequests().length);
}
 

public FailedEvaluatorBridge(final FailedEvaluator failedEvaluator,
                             final EvaluatorRequestor evaluatorRequestor,
                             final boolean blockedForAdditionalEvaluator,
                             final LoggingScopeFactory loggingScopeFactory,
                             final ActiveContextBridgeFactory activeContextBridgeFactory,
                             final Set<String> definedRuntimes) {
  this.jfailedEvaluator = failedEvaluator;
  this.evaluatorId = failedEvaluator.getId();
  this.evaluatorRequestorBridge =
      new EvaluatorRequestorBridge(evaluatorRequestor, blockedForAdditionalEvaluator, loggingScopeFactory,
              definedRuntimes);
  this.activeContextBridgeFactory = activeContextBridgeFactory;
}
 

public EvaluatorRequestorBridge(final EvaluatorRequestor evaluatorRequestor,
                                final boolean isBlocked,
                                final LoggingScopeFactory loggingScopeFactory,
                                final Set<String> definedRuntimes) {
  this.jevaluatorRequestor = evaluatorRequestor;
  this.clrEvaluatorsNumber = 0;
  this.isBlocked = isBlocked;
  this.loggingScopeFactory = loggingScopeFactory;
  this.definedRuntimes = definedRuntimes;
}
 

@Inject
private ContainerManager(@Parameter(JobConf.ScheduleSerThread.class) final int scheduleSerThread,
                         final EvaluatorRequestor evaluatorRequestor,
                         final MessageEnvironment messageEnvironment) {
  this.isTerminated = false;
  this.evaluatorRequestor = evaluatorRequestor;
  this.messageEnvironment = messageEnvironment;
  this.pendingContextIdToResourceSpec = new HashMap<>();
  this.pendingContainerRequestsByContainerType = new HashMap<>();
  this.evaluatorIdToResourceSpec = new HashMap<>();
  this.requestLatchByResourceSpecId = new HashMap<>();
  this.serializationExecutorService = Executors.newFixedThreadPool(scheduleSerThread);
}
 

@Inject
public ContainerManager(@Parameter(JobConf.ScheduleSerThread.class) final int scheduleSerThread,
                        final EvaluatorRequestor evaluatorRequestor,
                        final MessageEnvironment messageEnvironment) {
  this.evaluatorRequestor = evaluatorRequestor;
  this.messageEnvironment = messageEnvironment;
  this.persistentConnectionToMasterMap = new PersistentConnectionToMasterMap(messageEnvironment);
  this.executorsByContainerType = new HashMap<>();
  this.executorRepresenterMap = new HashMap<>();
  this.failedExecutorRepresenterMap = new HashMap<>();
  this.pendingContextIdToResourceSpec = new HashMap<>();
  this.pendingContainerRequestsByContainerType = new HashMap<>();
  this.requestLatchByResourceSpecId = new HashMap<>();
  this.serializationExecutorService = Executors.newFixedThreadPool(scheduleSerThread);
}
 

@Inject
OnDriverStartedAllocateOneInRack(
    final EvaluatorRequestor requestor,
    @Parameter(RackNameParameter.class) final String rackName) {
  this.requestor = requestor;
  this.rackName = rackName;
}
 

@Before
public void setUp() {

  final MessageEnvironment mockMsgEnv = mock(MessageEnvironment.class);
  when(mockMsgEnv.asyncConnect(anyString(), anyString())).thenReturn(mock(Future.class));
  containerManager = new ContainerManager(1, mock(EvaluatorRequestor.class), mockMsgEnv);
}
 

@Inject
public HeronMasterDriver(EvaluatorRequestor requestor,
                         final REEFFileNames fileNames,
                         @Parameter(Cluster.class) String cluster,
                         @Parameter(Role.class) String role,
                         @Parameter(TopologyName.class) String topologyName,
                         @Parameter(Environ.class) String env,
                         @Parameter(TopologyJar.class) String topologyJar,
                         @Parameter(TopologyPackageName.class) String topologyPackageName,
                         @Parameter(HeronCorePackageName.class) String heronCorePackageName,
                         @Parameter(HttpPort.class) int httpPort,
                         @Parameter(VerboseLogMode.class) boolean verboseMode)
    throws IOException {

  // REEF related initialization
  this.requestor = requestor;
  this.reefFileNames = fileNames;

  // Heron related initialization
  this.localHeronConfDir = ".";
  this.cluster = cluster;
  this.role = role;
  this.topologyName = topologyName;
  this.topologyPackageName = topologyPackageName;
  this.heronCorePackageName = heronCorePackageName;
  this.env = env;
  this.topologyJar = topologyJar;
  this.httpPort = httpPort;
  this.verboseMode = verboseMode;
  this.multiKeyWorkerMap = new MultiKeyWorkerMap();

  // This instance of Driver will be used for managing topology containers
  HeronMasterDriverProvider.setInstance(this);
}
 

@Inject
private MultipleCommGroupsDriver(final EvaluatorRequestor requestor,
                                 final GroupCommDriver groupCommDriver) {
  this.requestor = requestor;
  this.groupCommDriver = groupCommDriver;
  taskIds = new String[][]{
      {"MasterTask-1", "SlaveTask-1-1", "SlaveTask-1-2", "SlaveTask-1-3"},
      {"MasterTask-2", "SlaveTask-2-1"}
  };
  taskCounter = new AtomicInteger[]{new AtomicInteger(0), new AtomicInteger(0)};
  commGroupDriverList = new ArrayList<>(2);
  activeContextsToBeHandled = new ArrayList<>(2);
  initializeCommGroups();
}
 

@Inject
FileResourceTestDriver(@Parameter(FileResourceTestDriverConfiguration.FileNamesToExpect.class)
                       final Set<String> fileNamesToExpect,
                       final EvaluatorRequestor requestor,
                       final REEFFileNames fileNames) {
  this.fileNamesToExpect = fileNamesToExpect;
  this.requestor = requestor;
  this.fileNames = fileNames;
  this.localFolder = fileNames.getLocalFolder();
}
 

@Inject
private WatcherTestDriver(final EvaluatorRequestor evaluatorRequestor,
                          final TestEventStream testEventStream) {
  this.evaluatorRequestor = evaluatorRequestor;
  this.testEventStream = testEventStream;
  this.isFirstEvaluator = new AtomicBoolean(true);
  this.isFirstTask = new AtomicBoolean(true);
}
 

/**
 * Job driver constructor - instantiated via TANG.
 *
 * @param requestor evaluator requestor object used to create new evaluator containers.
 */
@Inject
private HelloDriver(final EvaluatorRequestor requestor) {
  this.requestor = requestor;
  LOG.log(Level.FINE, "Instantiated 'HelloDriver'");
}
 

@Before
public void setUp() throws InjectionException {
  final Configuration configuration = Tang.Factory.getTang().newConfigurationBuilder()
    .bindNamedParameter(JobConf.ScheduleSerThread.class, "1")
    .build();
  final Injector baseInjector = Tang.Factory.getTang().newInjector(configuration);
  baseInjector.bindVolatileInstance(EvaluatorRequestor.class, mock(EvaluatorRequestor.class));
  final Injector dispatcherInjector = LocalMessageDispatcher.forkInjector(baseInjector);
  final Injector injector = LocalMessageEnvironment.forkInjector(dispatcherInjector,
    MessageEnvironment.MASTER_COMMUNICATION_ID);

  final PlanRewriter planRewriter = mock(PlanRewriter.class);
  injector.bindVolatileInstance(PlanRewriter.class, planRewriter);

  injector.bindVolatileInstance(PubSubEventHandlerWrapper.class, mock(PubSubEventHandlerWrapper.class));
  final AtomicInteger executorCount = new AtomicInteger(0);
  injector.bindVolatileInstance(ClientRPC.class, mock(ClientRPC.class));
  injector.bindVolatileInstance(MetricManagerMaster.class, mock(MetricManagerMaster.class));
  injector.bindVolatileInstance(MetricMessageHandler.class, mock(MetricMessageHandler.class));
  injector.bindVolatileParameter(JobConf.DAGDirectory.class, EMPTY_DAG_DIRECTORY);
  injector.bindVolatileParameter(JobConf.JobId.class, "jobId");

  // Necessary for wiring up the message environments
  injector.bindVolatileInstance(Scheduler.class, injector.getInstance(BatchScheduler.class));
  injector.getInstance(RuntimeMaster.class);
  final BlockManagerMaster master = injector.getInstance(BlockManagerMaster.class);
  final MetricManagerWorker metricMessageSender = injector.getInstance(MetricManagerWorker.class);

  final Injector nameClientInjector = createNameClientInjector();
  nameClientInjector.bindVolatileParameter(JobConf.JobId.class, "data transfer test");

  this.master = master;
  final Pair<BlockManagerWorker, IntermediateDataIOFactory> pair1 = createWorker(
    EXECUTOR_ID_PREFIX + executorCount.getAndIncrement(), dispatcherInjector, nameClientInjector);
  this.worker1 = pair1.left();
  this.transferFactory = pair1.right();
  this.worker2 = createWorker(EXECUTOR_ID_PREFIX + executorCount.getAndIncrement(), dispatcherInjector,
    nameClientInjector).left();

  this.metricMessageSender = metricMessageSender;
}
 

@Inject
MockApplication(final Clock clock, final EvaluatorRequestor evaluatorRequestor) {
  this.clock = clock;
  this.evaluatorRequestor = evaluatorRequestor;
}