org.apache.mesos.Protos.ExecutorID Java Examples

/**
 * Setup testing target & sample data.
 */
public LaunchTaskTest()
{
  SlaveID slaveID = SlaveID.newBuilder().setValue("s1").build();
  this.sampleTaskInfo =
      TaskInfo.newBuilder()
              .setName("t1").setSlaveId(slaveID)
              .setTaskId(TaskID.newBuilder().setValue("id2"))
              .setExecutor(
                  ExecutorInfo.newBuilder()
                              .setExecutorId(ExecutorID.newBuilder().setValue("e1"))
                              .setCommand(CommandInfo.getDefaultInstance()))
              .build();
  this.sampleOffer =
      Offer.newBuilder()
           .setHostname("h1").setSlaveId(slaveID)
           .setId(OfferID.newBuilder().setValue("id1"))
           .setFrameworkId(FrameworkID.newBuilder().setValue("f1").build())
           .build();
  this.target = new LaunchTask(sampleTaskInfo, sampleOffer);
}
 

@Override
public void executorLost(final SchedulerDriver driver,
                         final Protos.ExecutorID executorId,
                         final Protos.SlaveID slaveId,
                         final int status) {
  final String diagnostics = "Executor Lost. executorid: "+executorId.getValue()+" slaveid: "+slaveId.getValue();
  final ResourceStatusEvent resourceStatus =
      ResourceStatusEventImpl.newBuilder()
          .setIdentifier(executorId.getValue())
          .setState(State.FAILED)
          .setExitCode(status)
          .setDiagnostics(diagnostics)
          .build();

  this.reefEventHandlers.onResourceStatus(resourceStatus);
}
 

ExecutorDriverContext(final String hostName,
                      final UPID slaveUpid,
                      final SlaveID slaveId,
                      final FrameworkID frameworkId,
                      final ExecutorID executorId)
                throws IOException
{
    this.slaveUpid = slaveUpid;
    this.slaveId = slaveId;
    this.frameworkId = frameworkId;
    this.executorId = executorId;

    this.driverUpid = UPID.fromParts(UUID.randomUUID().toString(),
        HostAndPort.fromParts(hostName, NetworkUtil.findUnusedPort()));
}
 

/**
 * Invoked when an executor sends a message. These messages are best
 * effort; do not expect a framework message to be retransmitted in
 * any reliable fashion.
 */
@Override
public void frameworkMessage(SchedulerDriver driver, ExecutorID executorId, SlaveID slaveId, byte[] data) {
	if (verbose) Gpr.debug("Scheduler: Framework message" //
			+ "\n\tExecutorId : " + executorId.getValue() //
			+ "\n\tSlaveId    : " + slaveId.getValue() //
			+ "\n\tMesssage   : '" + new String(data) + "'" //
	);
}
 

@Override
public void frameworkMessage(
    SchedulerDriver driver,
    ExecutorID executorID,
    SlaveID slave,
    byte[] data) {
  handler.handleMessage(convert(executorID), convert(slave));
}
 

@Override
public void frameworkMessage(SchedulerDriver schedDriver, ExecutorID executor,
        SlaveID slave, byte[] bytes) {
    try {
        LOG.log(Level.INFO,"Mesos framework executor"+executor+" on slave "+slave+" issued message: "+
            new String(bytes,"ascii"));
    } catch (UnsupportedEncodingException e) {
        LOG.log(Level.WARNING,"Mesos framework message missed due to bad encoding: ascii. "+e.getMessage());
    }
}
 

@Override
public void frameworkMessage(final SchedulerDriver driver,
                             final Protos.ExecutorID executorId,
                             final Protos.SlaveID slaveId,
                             final byte[] data) {
  LOG.log(Level.INFO, "Framework Message. driver: {0} executorId: {1} slaveId: {2} data: {3}",
      new Object[]{driver, executorId, slaveId, data});
}
 

@Override
public Status sendFrameworkMessage(final ExecutorID executorId, final SlaveID slaveId, final byte[] data)
{
    checkNotNull(executorId, "executorId is null");
    checkNotNull(slaveId, "slaveId is null");
    checkNotNull(data, "data is null");

    if (!context.isStateMachine(DRIVER_RUNNING)) {
        return context.getStateMachine();
    }

    final FrameworkToExecutorMessage message = FrameworkToExecutorMessage.newBuilder()
        .setFrameworkId(context.getFrameworkId())
        .setExecutorId(executorId)
        .setSlaveId(slaveId)
        .setData(ByteString.copyFrom(data)).build();

    // If the UPID of that slave is known (from the slave cache, then send the message
    // directly to the slave, otherwise to the master and let the master sort it out.
    if (context.containsSlave(message.getSlaveId())) {
        final UPID slave = context.getSlaveUPID(message.getSlaveId());
        eventBus.post(new RemoteMessageEnvelope(context.getDriverUPID(), slave, message));
    }
    else {
        eventBus.post(new RemoteMessageEnvelope(context.getDriverUPID(), context.getMasterUPID(), message));
    }

    return context.getStateMachine();
}
 

public JesosExecutorDriver(final Executor executor,
                           final UPID slaveUpid,
                           final SlaveID slaveId,
                           final FrameworkID frameworkId,
                           final ExecutorID executorId) throws IOException
{
    super(executor, slaveUpid, slaveId, frameworkId, executorId);
}
 

public JesosExecutorDriver(final Executor executor) throws IOException
{
    this(executor,
         UPID.create(System.getenv("MESOS_SLAVE_PID")),
         SlaveID.newBuilder().setValue(System.getenv("MESOS_SLAVE_ID")).build(),
         FrameworkID.newBuilder().setValue(System.getenv("MESOS_FRAMEWORK_ID")).build(),
         ExecutorID.newBuilder().setValue(System.getenv("MESOS_EXECUTOR_ID")).build());
}
 

private void stopExecutors(final String appName) {
    try {
        Collection<MesosStateService.ExecutorStateInfo> executorBriefInfo = mesosStateService.executors(appName);
        for (MesosStateService.ExecutorStateInfo each : executorBriefInfo) {
            producerManager.sendFrameworkMessage(ExecutorID.newBuilder().setValue(each.getId()).build(),
                    SlaveID.newBuilder().setValue(each.getSlaveId()).build(), "STOP".getBytes());
        }
    } catch (final JSONException ex) {
        throw new JobSystemException(ex);
    }
}
 

@Override
public void onEvent(ExecutorLostEvent event, long sequence, boolean endOfBatch) throws Exception {
  ExecutorID executorId = event.getExecutorId();
  SlaveID slaveId = event.getSlaveId();
  int exitStatus = event.getExitStatus();
  LOGGER.info("Executor {} of slave {} lost with exit status: {}", executorId, slaveId, exitStatus);
}
 

/**
* Returns a NodeTask given a ServiceResourceProfile and hostname
* 
* @param hostName
* @param profile
* @return
*/
public static NodeTask getNodeTask(String hostName, ServiceResourceProfile profile) {
  NodeTask task = new NodeTask(profile, new LikeConstraint(hostName, "host-[0-9]*.example.com"));
  task.setHostname(hostName);
  task.setTaskPrefix("nm");
  task.setSlaveId(SlaveID.newBuilder().setValue(profile.getName() + "-" + hostName).build());
  task.setExecutorInfo(ExecutorInfo.newBuilder().setExecutorId(ExecutorID.newBuilder().setValue("exec")).
       setCommand(org.apache.mesos.Protos.CommandInfo.newBuilder().setValue("command")).build());
  return task;
}
 

@Override
public void frameworkMessage(SchedulerDriver driver, ExecutorID executorId, SlaveID slaveId, byte[] data) {
}
 

@Override
public Status sendFrameworkMessage(ExecutorID executorId, SlaveID slaveId, byte[] data) {
  throw new UnsupportedOperationException();
}
 

@Override
public void executorLost(SchedulerDriver driver, ExecutorID executorId, SlaveID slaveId, int status) {
    logMesosCallbackError("Lost executor %s on slave %s with status=%s", executorId.getValue(), slaveId.getValue(), status);
}
 

@Override
public void executorLost(SchedulerDriver schedulerDriver, ExecutorID executorID, SlaveID slaveID,
    int status) {
  handler.handleLostExecutor(convert(executorID), convert(slaveID), status);
}
 

public Scheduler construct() {
    try {
        String uri = System.getProperty("org.apache.oodt.cas.resource.mesos.executor.uri","./oodt-executor.in");
        //Framework info
        FrameworkInfo.Builder frameworkBuilder = FrameworkInfo.newBuilder()
                    .setName("OODT Resource Manager Mesos Framework").setUser("")
                    .setId(FrameworkID.newBuilder().setValue("OODT-Resource Framework").build());
        FrameworkInfo framework = frameworkBuilder.build();
        ExecutorInfo executor = ExecutorInfo.newBuilder().setExecutorId(ExecutorID.newBuilder().setValue("OODT-Resource").build())
                .setCommand(CommandInfo.newBuilder().setValue(new File(uri).getCanonicalPath()).build())
                .setName("OODT Resource Manager Executor").build();
        SchedulerDriver driver = null;

        //Resource manager properties
        String batchmgrClassStr = "org.apache.oodt.cas.resource.batchmgr.MesosBatchManagerFactory";
        String monitorClassStr = "org.apache.oodt.cas.resource.monitor.MesosMonitorFactory";
        String jobQueueClassStr = System.getProperty("resource.jobqueue.factory","org.apache.oodt.cas.resource.jobqueue.JobStackJobQueueFactory");
        String ip = System.getProperty("resource.mesos.master.ip","127.0.0.1:5050");

        batch = (MesosBatchManager)GenericResourceManagerObjectFactory.getBatchmgrServiceFromFactory(batchmgrClassStr);
        mon = GenericResourceManagerObjectFactory.getMonitorServiceFromFactory(monitorClassStr);
        queue = GenericResourceManagerObjectFactory.getJobQueueServiceFromFactory(jobQueueClassStr);
        batch.setMonitor(mon);
        batch.setDriver(driver);
        batch.setJobRepository(queue.getJobRepository());

        LOG.log(Level.INFO,"Connecting to Mesos Master at: "+ip);
        System.out.println("Connecting to Mesos Master at: "+ip);
        ResourceMesosScheduler scheduler = new ResourceMesosScheduler(batch, executor, queue, mon);

        final MesosSchedulerDriver mesos = new MesosSchedulerDriver(scheduler, framework, ip);
        //Anonymous thread to run
        new Thread(new Runnable() {
            public void run() {
                int status = mesos.run() == Status.DRIVER_STOPPED ? 0 : 1;
                mesos.stop();
            }
        }).start();
        return scheduler;
    } catch(IOException ioe) {
        LOG.log(Level.SEVERE,"Exception detected: "+ioe.getMessage());
        ioLOG.log(Level.SEVERE, e.getMessage());
        throw new RuntimeException(ioe);
    }
}
 

@Override
public void frameworkMessage(SchedulerDriver driver, ExecutorID executorId, SlaveID slaveId, byte[] data) {
    logMesosCallbackError("Unexpected framework message: executorId=%s slaveID=%s, message=%s", executorId.getValue(), slaveId.getValue(), data);
}
 

@Override
public void executorLost(SchedulerDriver schedDriver, ExecutorID executor,SlaveID slave, int status) {
    //Tasks will have a "task lost" message automatically q.e.d no action necessary.
    //TODO: do we need to restart?
    LOG.log(Level.SEVERE,"Mesos executor "+executor+" on slave "+slave+" died with status "+status);
}
 

@Override
public void frameworkMessage(SchedulerDriver driver, ExecutorID executorId, SlaveID slaveId, byte[] data) { }
 

private ExecutorInfo.Builder getExecutorInfoBuilder(TopologyDetails details, String executorDataStr,
                                                    String executorName,
                                                    List<Resource> executorResources,
                                                    String extraConfig) {
  String configUri;

  configUri = getFullConfigUri();

  ExecutorInfo.Builder executorInfoBuilder = ExecutorInfo.newBuilder();

  executorInfoBuilder
    .setName(executorName)
    .setExecutorId(ExecutorID.newBuilder().setValue(details.getId()))
    .setData(ByteString.copyFromUtf8(executorDataStr))
    .addAllResources(executorResources);

  ICommandLineShim commandLineShim = CommandLineShimFactory.makeCommandLineShim(_container.isPresent(), extraConfig);
  /**
   *  _container.isPresent() might be slightly misleading at first blush. It is only checking whether or not
   *  CONF_MESOS_CONTAINER_DOCKER_IMAGE is set to a value other than null.
   */
  if (_container.isPresent()) {
    executorInfoBuilder.setCommand(CommandInfo.newBuilder()
                                              .addUris(URI.newBuilder().setValue(configUri))
                                              .setValue(commandLineShim.getCommandLine(details.getId())))
                       .setContainer(ContainerInfo.newBuilder()
                                                  .setType(ContainerInfo.Type.DOCKER)
                                                  .setDocker(ContainerInfo.DockerInfo.newBuilder()
                                                                                     .setImage(_container.get())
                                                                                     .setNetwork(ContainerInfo.DockerInfo.Network.HOST)
                                                                                     .setForcePullImage(true)
                                                                                     .build()
                                                  ).build());
  } else {
    executorInfoBuilder.setCommand(CommandInfo.newBuilder()
                                              .addUris(URI.newBuilder().setValue((String) mesosStormConf.get(CONF_EXECUTOR_URI)))
                                              .addUris(URI.newBuilder().setValue(configUri))
                                              .setValue(commandLineShim.getCommandLine(details.getId())));
  }

  return executorInfoBuilder;
}
 

@Override
public void executorLost(SchedulerDriver driver, ExecutorID executor, SlaveID slave, int status) {
  LOG.warn("Mesos Executor lost: executor: {} slave: {} status: {}", executor.getValue(), slave.getValue(), status);
}
 

/**
 * Greedily acquire resources by launching a Mesos Task(w/ our custom MesosExecutor) on REEF Evaluator request.
 * Either called from onResourceRequest(for a new request) or resourceOffers(for an outstanding request).
 * TODO[JIRA REEF-102]: reflect priority and rack/node locality specified in resourceRequestEvent.
 */
private synchronized void doResourceRequest(final ResourceRequestEvent resourceRequestEvent) {
  int tasksToLaunchCounter = resourceRequestEvent.getResourceCount();

  for (final Offer offer : this.offers.values()) {
    final int cpuSlots = getCpu(offer) / resourceRequestEvent.getVirtualCores().get();
    final int memSlots = getMemory(offer) / resourceRequestEvent.getMemorySize().get();
    final int taskNum = Math.min(Math.min(cpuSlots, memSlots), tasksToLaunchCounter);

    if (taskNum > 0 && satisfySlaveConstraint(resourceRequestEvent, offer)) {
      final List<TaskInfo> tasksToLaunch = new ArrayList<>();
      tasksToLaunchCounter -= taskNum;

      // Launch as many MesosTasks on the same node(offer) as possible to exploit locality.
      for (int j = 0; j < taskNum; j++) {
        final String id = offer.getId().getValue() + "-" + String.valueOf(j);
        final String executorLaunchCommand = getExecutorLaunchCommand(id, resourceRequestEvent.getMemorySize().get());

        final ExecutorInfo executorInfo = ExecutorInfo.newBuilder()
            .setExecutorId(ExecutorID.newBuilder()
                .setValue(id)
                .build())
            .setCommand(CommandInfo.newBuilder()
                .setValue(executorLaunchCommand)
                .addUris(URI.newBuilder().setValue(reefTarUri).build())
                .build())
            .build();

        final TaskInfo taskInfo = TaskInfo.newBuilder()
            .setTaskId(TaskID.newBuilder()
                .setValue(id)
                .build())
            .setName(id)
            .setSlaveId(offer.getSlaveId())
            .addResources(Resource.newBuilder()
                    .setName("mem")
                    .setType(Type.SCALAR)
                    .setScalar(Value.Scalar.newBuilder()
                            .setValue(resourceRequestEvent.getMemorySize().get())
                            .build())
                    .build())
            .addResources(Resource.newBuilder()
                    .setName("cpus")
                    .setType(Type.SCALAR)
                    .setScalar(Value.Scalar.newBuilder()
                            .setValue(resourceRequestEvent.getVirtualCores().get())
                            .build())
                    .build())
            .setExecutor(executorInfo)
            .build();

        tasksToLaunch.add(taskInfo);
        this.executorIdToLaunchedRequests.put(id, resourceRequestEvent);
      }

      final Filters filters = Filters.newBuilder().setRefuseSeconds(0).build();
      mesosMaster.launchTasks(Collections.singleton(offer.getId()), tasksToLaunch, filters);
    } else {
      mesosMaster.declineOffer(offer.getId());
    }
  }

  // the offers are no longer valid(all launched or declined)
  this.offers.clear();

  // Save leftovers that couldn't be launched
  outstandingRequests.add(ResourceRequestEventImpl.newBuilder()
      .mergeFrom(resourceRequestEvent)
      .setResourceCount(tasksToLaunchCounter)
      .build());
}
 

@Override
public void executorLost(SchedulerDriver driver, ExecutorID executorId, SlaveID slaveId, int status) { }
 

@Override
public void onEvent(FrameworkMessageEvent event, long sequence, boolean endOfBatch) throws Exception {
  ExecutorID executorId = event.getExecutorId();
  SlaveID slaveId = event.getSlaveId();
  LOGGER.info("Received framework message from executor {} of slave {}", executorId, slaveId);
}
 

ExecutorID getExecutorId()
{
    return executorId;
}
 

protected InternalExecutorDriver(final Executor executor,
                                 final UPID slaveUpid,
                                 final SlaveID slaveId,
                                 final FrameworkID frameworkId,
                                 final ExecutorID executorId) throws IOException
{
    this.executor = checkNotNull(executor, "executor is null");

    checkNotNull(slaveUpid, "slaveUpid is null");
    checkNotNull(slaveId, "slaveId is null");
    checkNotNull(frameworkId, "frameworkId is null");
    checkNotNull(executorId, "executorId is null");

    LOG.debug("Slave UPID:       %s", slaveUpid.asString());
    LOG.debug("Slave ID:         %s", slaveId.getValue());
    LOG.debug("Framework ID:     %s", frameworkId.getValue());
    LOG.debug("Executor ID:      %s", executorId.getValue());

    // Enforce using of the IP, when using the hostname this might "flap" between IPs which in turn
    // confuses the heck out of Mesos.
    final String hostName = NetworkUtil.findPublicIp();

    LOG.debug("Host name:        %s", hostName);

    this.context = new ExecutorDriverContext(hostName, slaveUpid, slaveId, frameworkId, executorId);

    this.eventBus = new ManagedEventBus("executor");

    this.localMessageProcessor = new LocalExecutorMessageProcessor(context, eventBus);

    // Closer closes in reverse registration order.

    // Close the callback executor last, so that everything that was still scheduled to be delivered to the framework still has a chance.
    this.callbackExecutor = closer.register(CloseableExecutors.decorate(Executors.newScheduledThreadPool(5, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("executor-callback-%d").build())));

    this.receiver = closer.register(new HttpProtocolReceiver(context.getDriverUPID(), ExecutorMessageEnvelope.class, eventBus));

    // The sender is closed before the receiver, so that possible responses are still caught
    this.sender = closer.register(new HttpProtocolSender(context.getDriverUPID()));

    // Make sure that the event bus is drained first at shutdown.
    closer.register(eventBus);
}
 

private ExecutorInfo getExecutorInfo() {
  FrameworkID id = Protos.FrameworkID.newBuilder().setValue("framework1").build();
  ExecutorID eid = Protos.ExecutorID.newBuilder().setValue("executor1").build();
  CommandInfo cm = Protos.CommandInfo.newBuilder().setValue("command").build();
  return ExecutorInfo.newBuilder().setFrameworkId(id).setExecutorId(eid).setCommand(cm).build();
}
 

@Override
public Status sendFrameworkMessage(ExecutorID executorId, SlaveID slaveId, byte[] data) {
  return null;
}