org.apache.mesos.Protos.FrameworkID Java Examples

public ESTaskStatus(SerializableState state, FrameworkID frameworkID, TaskInfo taskInfo, StatePath statePath) {
    if (state == null || taskInfo == null) {
        throw new InvalidParameterException("Cannot be null");
    } else if (frameworkID == null || frameworkID.getValue().isEmpty()) {
        throw new InvalidParameterException("FrameworkID cannot be null or empty");
    }
    this.state = state;
    this.frameworkID = frameworkID;
    this.taskInfo = taskInfo;
    this.statePath = statePath;
    // Write default status if it doesn't exist
    try {
        LOGGER.debug("Task status for " + taskInfo.getTaskId().getValue() + " exists, using old state: " + getStatus().getState());
    } catch (IllegalStateException | NullPointerException e) {
        LOGGER.debug("Task status for " + taskInfo.getTaskId().getValue() + " does not exist, or is not yet initialized, this must be a new task. Writing status.");
        setStatus(getDefaultStatus());
    }
}
 

private SchedulerState getSchedulerState() throws Exception {
  SchedulerState state = new SchedulerState(new MyriadFileSystemRMStateStore());
  idOne = Protos.TaskID.newBuilder().setValue("nt-1").build();
  idTwo = Protos.TaskID.newBuilder().setValue("nt-2").build();
  idThree = Protos.TaskID.newBuilder().setValue("nt-3").build();
  TreeMap<String, Long> ports = new TreeMap<>();

  state.addTask(idOne, new NodeTask(new ServiceResourceProfile("profile1", 0.2, 1024.0, ports), new LikeConstraint("localhost", "host-[0-9]*.example.com")));
  state.addTask(idTwo, new NodeTask(new ServiceResourceProfile("profile2", 0.4, 2048.0, ports), new LikeConstraint("localhost", "host-[0-9]*.example.com")));
  state.addTask(idThree, new NodeTask(new ServiceResourceProfile("profile3", 0.6, 3072.0, ports), new LikeConstraint("localhost", "host-[0-9]*.example.com")));

  state.setFrameworkId(FrameworkID.newBuilder().setValue("mock-framework").build());
  state.makeTaskActive(idOne);
  state.makeTaskPending(idTwo);
  state.makeTaskStaging(idThree);

  return state;
}
 

private FrameworkInfo.Builder createFrameworkBuilder() throws IOException {
  Number failoverTimeout = Optional.fromNullable((Number) mesosStormConf.get(CONF_MASTER_FAILOVER_TIMEOUT_SECS)).or(24 * 7 * 3600);
  String role = Optional.fromNullable((String) mesosStormConf.get(CONF_MESOS_ROLE)).or("*");
  Boolean checkpoint = Optional.fromNullable((Boolean) mesosStormConf.get(CONF_MESOS_CHECKPOINT)).or(false);
  String frameworkUser = Optional.fromNullable((String) mesosStormConf.get(CONF_MESOS_FRAMEWORK_USER)).or("");

  FrameworkInfo.Builder finfo = FrameworkInfo.newBuilder()
                                             .setName(frameworkName)
                                             .setFailoverTimeout(failoverTimeout.doubleValue())
                                             .setUser(frameworkUser)
                                             .setRole(role)
                                             .setCheckpoint(checkpoint);

  String id = _state.get(FRAMEWORK_ID);

  if (id != null) {
    finfo.setId(FrameworkID.newBuilder().setValue(id).build());
  }

  return finfo;
}
 

/**
 * Setup testing target & sample data.
 */
public LaunchTaskTest()
{
  SlaveID slaveID = SlaveID.newBuilder().setValue("s1").build();
  this.sampleTaskInfo =
      TaskInfo.newBuilder()
              .setName("t1").setSlaveId(slaveID)
              .setTaskId(TaskID.newBuilder().setValue("id2"))
              .setExecutor(
                  ExecutorInfo.newBuilder()
                              .setExecutorId(ExecutorID.newBuilder().setValue("e1"))
                              .setCommand(CommandInfo.getDefaultInstance()))
              .build();
  this.sampleOffer =
      Offer.newBuilder()
           .setHostname("h1").setSlaveId(slaveID)
           .setId(OfferID.newBuilder().setValue("id1"))
           .setFrameworkId(FrameworkID.newBuilder().setValue("f1").build())
           .build();
  this.target = new LaunchTask(sampleTaskInfo, sampleOffer);
}
 

@Inject
public MyriadDriver(final MyriadScheduler scheduler,
		final MyriadConfiguration cfg, final SchedulerState schedulerState) {
	this.scheduler = scheduler;
	FrameworkID frameworkId = schedulerState.getFrameworkId();
	Builder frameworkInfoBuilder = FrameworkInfo.newBuilder().setUser("")
			.setName(cfg.getFrameworkName())
			.setCheckpoint(cfg.getCheckpoint())
			.setFailoverTimeout(cfg.getFrameworkFailoverTimeout());
	if (frameworkId != null) {
		frameworkInfoBuilder.setId(frameworkId);
	}
	this.myriadFrameworkInfo = frameworkInfoBuilder.build();
	this.driver = new MesosSchedulerDriver(this.scheduler,
			this.myriadFrameworkInfo, cfg.getMesosMaster());
}
 

public JesosExecutorDriver(final Executor executor) throws IOException
{
    this(executor,
         UPID.create(System.getenv("MESOS_SLAVE_PID")),
         SlaveID.newBuilder().setValue(System.getenv("MESOS_SLAVE_ID")).build(),
         FrameworkID.newBuilder().setValue(System.getenv("MESOS_FRAMEWORK_ID")).build(),
         ExecutorID.newBuilder().setValue(System.getenv("MESOS_EXECUTOR_ID")).build());
}
 

public void doRegistration(final SchedulerDriver driver, Protos.FrameworkID id) {
  _driver = driver;
  // Now that we've set the driver, we can create our scheduler
  _stormScheduler = new StormSchedulerImpl(_driver);

  _state.put(FRAMEWORK_ID, id.getValue());
  _offers = new HashMap<Protos.OfferID, Protos.Offer>();

  if (_enabledLogviewerSidecar) {

    _timer.scheduleAtFixedRate(new TimerTask() {
      @Override
      public void run() {
        // performing "explicit" reconciliation; master will respond with the latest state for all logviewer tasks
        // in the framework scheduler's statusUpdate() method
        List<TaskStatus> taskStatuses = new ArrayList<TaskStatus>();
        List<String> logviewerPaths = _zkClient.getChildren(_logviewerZkDir);
        if (logviewerPaths == null) {
          _driver.reconcileTasks(taskStatuses);
          return;
        }
        for (String path : logviewerPaths) {
          TaskID logviewerTaskId = TaskID.newBuilder()
                                         .setValue(new String(_zkClient.getNodeData(String.format("%s/%s", _logviewerZkDir, path))))
                                         .build();
          TaskStatus logviewerTaskStatus = TaskStatus.newBuilder()
                                                     .setTaskId(logviewerTaskId)
                                                     .setState(TaskState.TASK_RUNNING)
                                                     .build();
          taskStatuses.add(logviewerTaskStatus);
        }
        _driver.reconcileTasks(taskStatuses);
        LOG.info("Performing task reconciliation between scheduler and master on following tasks: {}", taskStatusListToTaskIDsString(taskStatuses));
      }
    }, 0, TASK_RECONCILIATION_INTERVAL); // reconciliation performed every 5 minutes
  }
}
 

@Override
public void registered(final SchedulerDriver driver, FrameworkID id, MasterInfo masterInfo) {
  mesosNimbus.doRegistration(driver, id);

  // Completed registration, let anything waiting for us to do so continue
  _registeredLatch.countDown();
}
 

ExecutorDriverContext(final String hostName,
                      final UPID slaveUpid,
                      final SlaveID slaveId,
                      final FrameworkID frameworkId,
                      final ExecutorID executorId)
                throws IOException
{
    this.slaveUpid = slaveUpid;
    this.slaveId = slaveId;
    this.frameworkId = frameworkId;
    this.executorId = executorId;

    this.driverUpid = UPID.fromParts(UUID.randomUUID().toString(),
        HostAndPort.fromParts(hostName, NetworkUtil.findUnusedPort()));
}
 

void setFrameworkId(final FrameworkID frameworkId)
{
    checkNotNull(frameworkId, "frameworkId is null");

    frameworkInfo.set(FrameworkInfo.newBuilder(frameworkInfo.get())
        .setId(frameworkId)
        .build());
}
 

@Subscribe
public void frameworkReregistered(final FrameworkReregisteredMessageEnvelope envelope)
{
    checkState(envelope.getRecipient().equals(context.getDriverUPID()), "Received a remote message for local delivery");

    final FrameworkReregisteredMessage frameworkReregisteredMessage = envelope.getMessage();

    if (!masterIsValid(frameworkReregisteredMessage.getMasterInfo())) {
        return;
    }

    final FrameworkID frameworkId = frameworkReregisteredMessage.getFrameworkId();

    checkState(frameworkId != null, "Received null framework reregistration message!");
    checkState(frameworkId.equals(context.getFrameworkId()), "Received framework reregistration for %s but expected %s", frameworkId.getValue(), context.getFrameworkId().getValue());

    context.connected();
    context.setFailover(false);

    eventBus.post(new SchedulerCallback() {
        @Override
        public Runnable getCallback(final Scheduler scheduler, final SchedulerDriver schedulerDriver)
        {
            return new Runnable() {
                @Override
                public void run()
                {
                    scheduler.reregistered(schedulerDriver, context.getMaster());
                }
            };
        }
    });
}
 

@Subscribe
public void frameworkRegistered(final FrameworkRegisteredMessageEnvelope envelope)
{
    checkState(envelope.getRecipient().equals(context.getDriverUPID()), "Received a remote message for local delivery");

    final FrameworkRegisteredMessage frameworkRegisteredMessage = envelope.getMessage();

    if (!masterIsValid(frameworkRegisteredMessage.getMasterInfo())) {
        return;
    }

    final FrameworkID frameworkId = frameworkRegisteredMessage.getFrameworkId();

    context.connected();
    context.setFailover(false);
    context.setFrameworkId(frameworkId);

    eventBus.post(new SchedulerCallback() {
        @Override
        public Runnable getCallback(final Scheduler scheduler, final SchedulerDriver schedulerDriver)
        {
            return new Runnable() {
                @Override
                public void run()
                {
                    scheduler.registered(schedulerDriver, frameworkId, context.getMaster());
                }
            };
        }
    });
}
 

public JesosExecutorDriver(final Executor executor,
                           final UPID slaveUpid,
                           final SlaveID slaveId,
                           final FrameworkID frameworkId,
                           final ExecutorID executorId) throws IOException
{
    super(executor, slaveUpid, slaveId, frameworkId, executorId);
}
 

private MyriadOperations initialize() throws Exception {
  resetStoreState();
  SchedulerState sState = TestObjectFactory.getSchedulerState(cfg, "tmp/myriad-operations-test");
  sState.setFrameworkId(FrameworkID.newBuilder().setValue("mock-framework").build());
  AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> scheduler = TestObjectFactory.getYarnScheduler();
  MyriadDriverManager manager = TestObjectFactory.getMyriadDriverManager();
  MyriadWebServer webServer = TestObjectFactory.getMyriadWebServer(cfg);
  CompositeInterceptor registry = new CompositeInterceptor();
  LeastAMNodesFirstPolicy policy = new LeastAMNodesFirstPolicy(registry, scheduler, sState);

  manager.startDriver();

  return new MyriadOperations(cfg, sState, policy, manager, webServer, generateRMContext(scheduler));
}
 

private List<Offer> generateOffers(List<Resource> resources) {
  FrameworkID fidOne = Protos.FrameworkID.newBuilder().setValue("framework-1").build();
  FrameworkID fidTwo = Protos.FrameworkID.newBuilder().setValue("framework-2").build();
  FrameworkID fidThree = Protos.FrameworkID.newBuilder().setValue("framework-3").build();
  FrameworkID fidFour = Protos.FrameworkID.newBuilder().setValue("framework-4").build();

  OfferID oidOne = Protos.OfferID.newBuilder().setValue("offer-1").build();
  OfferID oidTwo = Protos.OfferID.newBuilder().setValue("offer-2").build(); 
  OfferID oidThree = Protos.OfferID.newBuilder().setValue("offer-3").build(); 
  OfferID oidFour = Protos.OfferID.newBuilder().setValue("offer-4").build(); 

  SlaveID sidOne = Protos.SlaveID.newBuilder().setValue("slave-1").build();
  SlaveID sidTwo = Protos.SlaveID.newBuilder().setValue("slave-2").build();
  SlaveID sidThree = Protos.SlaveID.newBuilder().setValue("slave-3").build();
  SlaveID sidFour = Protos.SlaveID.newBuilder().setValue("slave-4").build();

  Offer offerOne = Protos.Offer.newBuilder().setFrameworkId(fidOne).setHostname("10.0.0.1").setId(oidOne).setSlaveId(sidOne).
      addResources(resources.get(0)).addResources(resources.get(1)).build();
  Offer offerTwo = Protos.Offer.newBuilder().setFrameworkId(fidTwo).setHostname("10.0.0.2").setId(oidTwo).setSlaveId(sidTwo).
      addResources(resources.get(2)).addResources(resources.get(3)).build();
  Offer offerThree = Protos.Offer.newBuilder().setFrameworkId(fidThree).setHostname("10.0.0.3").setId(oidThree).setSlaveId(sidThree).
      addResources(resources.get(0)).addResources(resources.get(3)).build();
  Offer offerFour = Protos.Offer.newBuilder().setFrameworkId(fidFour).setHostname("10.0.0.4").setId(oidFour).setSlaveId(sidFour).
      addResources(resources.get(2)).addResources(resources.get(1)).build();

  return Lists.newArrayList(offerOne, offerTwo, offerThree, offerFour);
}
 

public static Offer getOffer(String host, String slaveId, String frameworkId, String offerId, double cpuCores, double memory) {
  Protos.SlaveID sid = SlaveID.newBuilder().setValue(slaveId).build();
  Protos.FrameworkID fid = FrameworkID.newBuilder().setValue(frameworkId).build();
  Protos.Value.Scalar cores = Protos.Value.Scalar.newBuilder().setValue(cpuCores).build();
  Protos.Value.Scalar mem = Protos.Value.Scalar.newBuilder().setValue(memory).build();
  Protos.Resource cpuResource = Protos.Resource.newBuilder().setName("cpus").setScalar(cores).setType(Type.SCALAR).build();
  Protos.Resource memResource  = Protos.Resource.newBuilder().setName("mem").setScalar(mem).setType(Type.SCALAR).build();
  return Protos.Offer.newBuilder().setHostname(host).setId(OfferID.newBuilder().setValue(offerId)).
      setSlaveId(sid).setFrameworkId(fid).addResources(cpuResource).addResources(memResource).build();  
}
 

@Override
public void registered(
    SchedulerDriver driver,
    final FrameworkID frameworkId,
    MasterInfo masterInfo) {
  handler.handleRegistration(convert(frameworkId), convert(masterInfo));
  isRegistered = true;
}
 

@Test
public void testFrameworkIDToFromByteBuffer() throws Exception {
  ByteBuffer bb = ByteBufferSupport.toByteBuffer(getExecutorInfo().getFrameworkId());
  FrameworkID id = ByteBufferSupport.toFrameworkID(bb);

  assertEquals(getExecutorInfo().getFrameworkId(), id);
}
 

@Override
public Status start() {
  assertNotStopped();

  Futures.getUnchecked(schedulerFuture).registered(this,
      FrameworkID.newBuilder().setValue("local").build(),
      MasterInfo.getDefaultInstance());

  eventBus.post(new Started());

  executor.scheduleAtFixedRate(
      () -> {
        List<Offer> allOffers;
        synchronized (sentOffers) {
          synchronized (idleOffers) {
            sentOffers.putAll(idleOffers);
            allOffers = ImmutableList.copyOf(idleOffers.values());
            idleOffers.clear();
          }
        }

        if (allOffers.isEmpty()) {
          LOG.info("All offers consumed, suppressing offer cycle.");
        } else {
          Futures.getUnchecked(schedulerFuture).resourceOffers(this, allOffers);
        }
      },
      1,
      5,
      TimeUnit.SECONDS);

  return Status.DRIVER_RUNNING;
}
 

@Test
public void testSetAndGetFrameworkITask() throws Exception {
  MockState mState = new MockState();
  mState.store(new MockVariable().setName("frameworkId").setValue(FrameworkID.newBuilder().setValue("mock-framework").build().toByteArray()));

  MyriadState state = new MyriadState(mState);
  state.setFrameworkId(FrameworkID.newBuilder().setValue("mock-framework").build());

  assertEquals("mock-framework", state.getFrameworkID().getValue());
}
 

@Override
public void registered(SchedulerDriver driver, FrameworkID frameworkID, MasterInfo masterInfo) {
    logMesosCallbackInfo("Mesos registered: %s, ID=%s, masterInfo=%s", driver, frameworkID.getValue(), masterInfo.getId());
    initializeNewDriver(driver);
    numMesosRegistered.increment();
    connected = true;
}
 

@Override
public void registered(SchedulerDriver driver, FrameworkID frameworkId, MasterInfo masterInfo) { }
 

@Override
public void registered(SchedulerDriver schedDriver, FrameworkID framework,
        MasterInfo masterInfo) {
    LOG.log(Level.INFO,"Mesos framework registered: "+framework.getValue()+" with master: "+masterInfo.getId());
}
 

public Scheduler construct() {
    try {
        String uri = System.getProperty("org.apache.oodt.cas.resource.mesos.executor.uri","./oodt-executor.in");
        //Framework info
        FrameworkInfo.Builder frameworkBuilder = FrameworkInfo.newBuilder()
                    .setName("OODT Resource Manager Mesos Framework").setUser("")
                    .setId(FrameworkID.newBuilder().setValue("OODT-Resource Framework").build());
        FrameworkInfo framework = frameworkBuilder.build();
        ExecutorInfo executor = ExecutorInfo.newBuilder().setExecutorId(ExecutorID.newBuilder().setValue("OODT-Resource").build())
                .setCommand(CommandInfo.newBuilder().setValue(new File(uri).getCanonicalPath()).build())
                .setName("OODT Resource Manager Executor").build();
        SchedulerDriver driver = null;

        //Resource manager properties
        String batchmgrClassStr = "org.apache.oodt.cas.resource.batchmgr.MesosBatchManagerFactory";
        String monitorClassStr = "org.apache.oodt.cas.resource.monitor.MesosMonitorFactory";
        String jobQueueClassStr = System.getProperty("resource.jobqueue.factory","org.apache.oodt.cas.resource.jobqueue.JobStackJobQueueFactory");
        String ip = System.getProperty("resource.mesos.master.ip","127.0.0.1:5050");

        batch = (MesosBatchManager)GenericResourceManagerObjectFactory.getBatchmgrServiceFromFactory(batchmgrClassStr);
        mon = GenericResourceManagerObjectFactory.getMonitorServiceFromFactory(monitorClassStr);
        queue = GenericResourceManagerObjectFactory.getJobQueueServiceFromFactory(jobQueueClassStr);
        batch.setMonitor(mon);
        batch.setDriver(driver);
        batch.setJobRepository(queue.getJobRepository());

        LOG.log(Level.INFO,"Connecting to Mesos Master at: "+ip);
        System.out.println("Connecting to Mesos Master at: "+ip);
        ResourceMesosScheduler scheduler = new ResourceMesosScheduler(batch, executor, queue, mon);

        final MesosSchedulerDriver mesos = new MesosSchedulerDriver(scheduler, framework, ip);
        //Anonymous thread to run
        new Thread(new Runnable() {
            public void run() {
                int status = mesos.run() == Status.DRIVER_STOPPED ? 0 : 1;
                mesos.stop();
            }
        }).start();
        return scheduler;
    } catch(IOException ioe) {
        LOG.log(Level.SEVERE,"Exception detected: "+ioe.getMessage());
        ioLOG.log(Level.SEVERE, e.getMessage());
        throw new RuntimeException(ioe);
    }
}
 

public void setFrameworkId(FrameworkID frameworkId) {
	this.frameworkId = frameworkId;
}
 

public FrameworkID getFrameworkId() {
	return frameworkId;
}
 

FrameworkID getFrameworkId()
{
    return frameworkId;
}
 

protected InternalExecutorDriver(final Executor executor,
                                 final UPID slaveUpid,
                                 final SlaveID slaveId,
                                 final FrameworkID frameworkId,
                                 final ExecutorID executorId) throws IOException
{
    this.executor = checkNotNull(executor, "executor is null");

    checkNotNull(slaveUpid, "slaveUpid is null");
    checkNotNull(slaveId, "slaveId is null");
    checkNotNull(frameworkId, "frameworkId is null");
    checkNotNull(executorId, "executorId is null");

    LOG.debug("Slave UPID:       %s", slaveUpid.asString());
    LOG.debug("Slave ID:         %s", slaveId.getValue());
    LOG.debug("Framework ID:     %s", frameworkId.getValue());
    LOG.debug("Executor ID:      %s", executorId.getValue());

    // Enforce using of the IP, when using the hostname this might "flap" between IPs which in turn
    // confuses the heck out of Mesos.
    final String hostName = NetworkUtil.findPublicIp();

    LOG.debug("Host name:        %s", hostName);

    this.context = new ExecutorDriverContext(hostName, slaveUpid, slaveId, frameworkId, executorId);

    this.eventBus = new ManagedEventBus("executor");

    this.localMessageProcessor = new LocalExecutorMessageProcessor(context, eventBus);

    // Closer closes in reverse registration order.

    // Close the callback executor last, so that everything that was still scheduled to be delivered to the framework still has a chance.
    this.callbackExecutor = closer.register(CloseableExecutors.decorate(Executors.newScheduledThreadPool(5, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("executor-callback-%d").build())));

    this.receiver = closer.register(new HttpProtocolReceiver(context.getDriverUPID(), ExecutorMessageEnvelope.class, eventBus));

    // The sender is closed before the receiver, so that possible responses are still caught
    this.sender = closer.register(new HttpProtocolSender(context.getDriverUPID()));

    // Make sure that the event bus is drained first at shutdown.
    closer.register(eventBus);
}
 

FrameworkID getFrameworkId()
{
    return frameworkInfo.get().getId();
}
 

private void doLaunchTasks(final LaunchTasksMessage message)
{
    final MasterInfo masterInfo = context.connectedMaster();

    if (masterInfo == null) {
        loseAllTasks(message.getTasksList(), "Master disconnected");
        return;
    }

    final ImmutableList.Builder<TaskInfo> builder = ImmutableList.builder();

    for (TaskInfo taskInfo : message.getTasksList()) {
        if (taskInfo.hasExecutor() == taskInfo.hasCommand()) {
            loseTask(taskInfo, "TaskInfo must have either an 'executor' or a 'command'");
            continue; // for(...
        }

        if (taskInfo.hasExecutor()) {
            if (taskInfo.getExecutor().hasFrameworkId()) {
                final FrameworkID executorFrameworkId = taskInfo.getExecutor().getFrameworkId();
                if (!executorFrameworkId.equals(context.getFrameworkId())) {
                    loseTask(taskInfo, format("ExecutorInfo has an invalid FrameworkID (Actual: %s vs Expected: %s)", executorFrameworkId.getValue(), context.getFrameworkId().getValue()));
                    continue; // for(...
                }
            }
            else {
                // Executor present but not framework id. Set the framework id.
                taskInfo = TaskInfo.newBuilder(taskInfo)
                    .setExecutor(ExecutorInfo.newBuilder(taskInfo.getExecutor()).setFrameworkId(context.getFrameworkId()))
                    .build();
            }
        }

        builder.add(taskInfo);
    }

    final List<TaskInfo> launchTasks = builder.build();

    for (final OfferID offer : message.getOfferIdsList()) {
        if (!context.hasOffers(offer)) {
            LOG.warn("Unknown offer %s ignored!", offer.getValue());
        }

        for (final TaskInfo launchTask : launchTasks) {
            if (context.hasOffer(offer, launchTask.getSlaveId())) {
                context.addSlave(launchTask.getSlaveId(), context.getOffer(offer, launchTask.getSlaveId()));
            }
        }
        context.removeAllOffers(offer);
    }

    final LaunchTasksMessage launchMessage = LaunchTasksMessage.newBuilder(message)
        .setFrameworkId(context.getFrameworkId())
        .clearTasks()
        .addAllTasks(launchTasks)
        .build();

    eventBus.post(new RemoteMessageEnvelope(context.getDriverUPID(), context.getMasterUPID(), launchMessage));
}