org.apache.mesos.Protos.Offer Java Examples

The following examples show how to use org.apache.mesos.Protos.Offer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AttributeRuleTest.java    From dcos-commons with Apache License 2.0 6 votes vote down vote up
@Test
public void testAnyMatchesRegex() {
    Offer o = getOfferWithResources()
            .addAttributes(ATTR_TEXT)
            .addAttributes(ATTR_SCALAR)
            .addAttributes(ATTR_RANGES)
            .addAttributes(ATTR_SET)
            .build();
    assertTrue(AttributeRuleFactory.getInstance().require(RegexMatcher.create(ATTR_TEXT_REGEX))
            .filter(o, POD_INSTANCE, Collections.emptyList()).isPassing());
    assertTrue(AttributeRuleFactory.getInstance().require(RegexMatcher.create(ATTR_SCALAR_REGEX))
            .filter(o, POD_INSTANCE, Collections.emptyList()).isPassing());
    assertTrue(AttributeRuleFactory.getInstance().require(RegexMatcher.create(ATTR_RANGES_REGEX))
            .filter(o, POD_INSTANCE, Collections.emptyList()).isPassing());
    assertTrue(AttributeRuleFactory.getInstance().require(RegexMatcher.create(ATTR_SET_REGEX))
            .filter(o, POD_INSTANCE, Collections.emptyList()).isPassing());
}
 
Example #2
Source File: MaxPerAttributeRule.java    From dcos-commons with Apache License 2.0 6 votes vote down vote up
@Override
public EvaluationOutcome filter(
    Offer offer,
    PodInstance podInstance,
    Collection<TaskInfo> tasks)
{
  if (isAcceptable(offer, podInstance, tasks)) {
    return EvaluationOutcome.pass(
        this,
        "Fits within limit of %d tasks matching filter '%s' on this agent with attribute: %s",
        max, taskFilter.toString(), attributeMatcher.toString())
        .build();
  } else {
    return EvaluationOutcome.fail(
        this,
        "Reached greater than %d tasks matching filter '%s' on this agent with attribute: %s",
        max,
        taskFilter.toString(),
        attributeMatcher.toString())
        .build();
  }
}
 
Example #3
Source File: AttributeRuleTest.java    From dcos-commons with Apache License 2.0 6 votes vote down vote up
@Test
public void testExactMatchesString() {
    Offer.Builder o = getOfferWithResources()
            .addAttributes(ATTR_TEXT);
    assertTrue(AttributeRuleFactory.getInstance().require(ExactMatcher.create(AttributeStringUtils.toString(ATTR_TEXT)))
            .filter(o.build(), POD_INSTANCE, Collections.emptyList()).isPassing());

    o = getOfferWithResources()
            .addAttributes(ATTR_SCALAR);
    assertTrue(AttributeRuleFactory.getInstance().require(ExactMatcher.create(AttributeStringUtils.toString(ATTR_SCALAR)))
            .filter(o.build(), POD_INSTANCE, Collections.emptyList()).isPassing());

    o = getOfferWithResources()
            .addAttributes(ATTR_RANGES);
    assertTrue(AttributeRuleFactory.getInstance().require(ExactMatcher.create(AttributeStringUtils.toString(ATTR_RANGES)))
            .filter(o.build(), POD_INSTANCE, Collections.emptyList()).isPassing());

    o = getOfferWithResources()
            .addAttributes(ATTR_SET);
    assertTrue(AttributeRuleFactory.getInstance().require(ExactMatcher.create(AttributeStringUtils.toString(ATTR_SET)))
            .filter(o.build(), POD_INSTANCE, Collections.emptyList()).isPassing());
}
 
Example #4
Source File: AgentRule.java    From dcos-commons with Apache License 2.0 6 votes vote down vote up
@Override
public EvaluationOutcome filter(
    Offer offer,
    PodInstance podInstance,
    Collection<TaskInfo> tasks)
{
  if (offer.getSlaveId().getValue().equals(agentId)) {
    return EvaluationOutcome.pass(this, "Offer matches required Agent ID '%s'", agentId).build();
  } else {
    // agent mismatch: return empty offer
    return EvaluationOutcome.fail(
        this,
        "Offer lacks required Agent ID. Wanted: '%s' Got: '%s'",
        agentId,
        offer.getSlaveId().getValue())
        .build();
  }
}
 
Example #5
Source File: StormSchedulerImplTest.java    From storm with Apache License 2.0 6 votes vote down vote up
@Before
public void initialize() {
  driver = null;
  stormSchedulerImpl = new StormSchedulerImpl(driver);
  Map<String, Object> mesosStormConf = new HashMap<>();
  stormSchedulerImpl.prepare(mesosStormConf);

  map = new HashMap<OfferID, Offer>();

  topologiesMissingAssignments = new HashSet<>();
  topologiesMissingAssignments.add("test-topology1-65-1442255385");
  topologiesMissingAssignments.add("test-topology1-65-1442255385");

  existingSupervisors = new ArrayList<>();
  existingSupervisors.add(new SupervisorDetails(MesosCommon.supervisorId(sampleFrameworkName, sampleHost, "test-topology1-65-1442255385"), sampleHost, null, null));
  existingSupervisors.add(new SupervisorDetails(MesosCommon.supervisorId(sampleFrameworkName, sampleHost, "test-topology10-65-1442255385"), sampleHost, null, null));

  topologyMap = new HashMap<>();
  topologyMap.put(sampleTopologyId, TestUtils.constructTopologyDetails(sampleTopologyId, 1, 0.1, 100));
  topologies = new Topologies(topologyMap);

  mesosWorkerSlotMap = new HashMap<>();
}
 
Example #6
Source File: OfferUtils.java    From incubator-myriad with Apache License 2.0 6 votes vote down vote up
/**
 * Transforms a collection of mesos offers into {@link Resource}.
 *
 * @param offers collection of mesos offers
 * @return a single resource object equivalent to the cumulative sum of mesos offers
 */
public static Resource getYarnResourcesFromMesosOffers(Collection<Offer> offers) {
  double cpus = 0.0;
  double mem = 0.0;

  for (Protos.Offer offer : offers) {
    for (Protos.Resource resource : offer.getResourcesList()) {
      if (resource.getName().equalsIgnoreCase("cpus")) {
        cpus += resource.getScalar().getValue();
      } else if (resource.getName().equalsIgnoreCase("mem")) {
        mem += resource.getScalar().getValue();
      }
    }
  }
  return Resource.newInstance((int) mem, (int) cpus);
}
 
Example #7
Source File: NMHeartBeatHandlerTest.java    From incubator-myriad with Apache License 2.0 6 votes vote down vote up
@Test
public void testIncrementNodeCapacityUnderCapacity() throws Exception {
  resetNodeTotalCapability(nodeOne, 0, 0);
  resetNodeTotalCapability(nodeTwo, 2, 512);
  Offer offerOne = TestObjectFactory.getOffer("localhost-one", "slave-one", "mock", "offer-one", 1.0, 512.0);
  Offer offerTwo = TestObjectFactory.getOffer("localhost-two", "slave-two", "mock", "offer-two", 3.0, 1024.0);
  olManager.addOffers(offerOne);
  olManager.addOffers(offerTwo);
      
  RMNodeStatusEvent eventOne = getRMStatusEvent(nodeOne);
  handler.beforeRMNodeEventHandled(eventOne, context);
  RMNodeStatusEvent eventTwo = getRMStatusEvent(nodeTwo);
  handler.beforeRMNodeEventHandled(eventTwo, context);
  
  assertEquals(512, nodeOne.getTotalCapability().getMemory());
  assertEquals(1, nodeOne.getTotalCapability().getVirtualCores());
  assertEquals(1024, nodeTwo.getTotalCapability().getMemory());
  assertEquals(3, nodeTwo.getTotalCapability().getVirtualCores());
}
 
Example #8
Source File: NMHeartBeatHandlerTest.java    From incubator-myriad with Apache License 2.0 6 votes vote down vote up
@Test
public void testIncrementNodeCapacityOverCapacity() throws Exception {
  resetNodeTotalCapability(nodeOne, 1, 512);
  resetNodeTotalCapability(nodeTwo, 2, 2048);
  
  //Test over memory upper limit
  Offer offerOne = TestObjectFactory.getOffer("localhost-one", "slave-one", "mock", "offer-one", 0.2, 3072.0);
  //Test over CPU cores upper limit
  Offer offerTwo = TestObjectFactory.getOffer("localhost-two", "slave-two", "mock", "offer-two", 8.0, 1024.0);
  olManager.addOffers(offerOne);
  olManager.addOffers(offerTwo);

  RMNodeStatusEvent eventOne = getRMStatusEvent(nodeOne);  
  handler.beforeRMNodeEventHandled(eventOne, context);
  RMNodeStatusEvent eventTwo = getRMStatusEvent(nodeTwo);
  handler.beforeRMNodeEventHandled(eventTwo, context);
  
  assertEquals(512, nodeOne.getTotalCapability().getMemory());
  assertEquals(1, nodeOne.getTotalCapability().getVirtualCores()); 
  assertEquals(2048, nodeTwo.getTotalCapability().getMemory());
  assertEquals(2, nodeTwo.getTotalCapability().getVirtualCores());
}
 
Example #9
Source File: InvalidPlacementRule.java    From dcos-commons with Apache License 2.0 6 votes vote down vote up
@Override
public EvaluationOutcome filter(
    Offer offer,
    PodInstance podInstance,
    Collection<TaskInfo> tasks)
{
  return EvaluationOutcome
      .fail(
          this,
          String.format(
              "Invalid placement constraints for %s: %s",
              podInstance.getName(),
              constraints
          )
      ).build();
}
 
Example #10
Source File: MesosResourcePoolTest.java    From dcos-commons with Apache License 2.0 6 votes vote down vote up
@Test
public void testConsumeUnreservedAtomicResourceUnmatchedProfile2() {
    Resource offerResource = ResourceTestUtils.getUnreservedMountVolume(1000, Optional.empty());
    VolumeSpec spec = DefaultVolumeSpec.createMountVolume(
            ValueUtils.getValue(offerResource).getScalar().getValue(),
            TestConstants.CONTAINER_PATH,
            Arrays.asList("foo", "bar"),
            TestConstants.ROLE,
            Constants.ANY_ROLE,
            TestConstants.PRINCIPAL);
    Offer offer = OfferTestUtils.getOffer(offerResource);
    MesosResourcePool pool = new MesosResourcePool(offer, Optional.of(Constants.ANY_ROLE));

    Assert.assertEquals(1, pool.getUnreservedAtomicPool().size());
    Assert.assertFalse(pool.consumeAtomic(offerResource.getName(), spec).isPresent());
}
 
Example #11
Source File: MesosSchedulerCallbackHandler.java    From titus-control-plane with Apache License 2.0 6 votes vote down vote up
private boolean validateOfferResources(Offer offer) {
    for (Protos.Resource resource : offer.getResourcesList()) {
        if ("cpus".equals(resource.getName())) {
            final double cpus = resource.getScalar().getValue();
            if (cpus < 0.1) {
                logMesosCallbackInfo("Declining offer: %s due to too few CPUs in offer from %s: %s", offer.getId().getValue(), offer.getHostname(), cpus);
                return false;
            }
        } else if ("mem".equals(resource.getName())) {
            double memoryMB = resource.getScalar().getValue();
            if (memoryMB < 1) {
                logMesosCallbackInfo("Declining offer: %s due to too few memory in offer from %s: %s", offer.getId().getValue(), offer.getHostname(), memoryMB);
                return false;
            }
        }
    }
    return true;
}
 
Example #12
Source File: SimulatedTitusAgent.java    From titus-control-plane with Apache License 2.0 6 votes vote down vote up
private Offer createOfferForAvailableResources() {
    Protos.OfferID offerId = Protos.OfferID.newBuilder().setValue(slaveId.getValue() + "_O_" + offerIdx++).build();
    String enis = "ResourceSet-ENIs-7-" + networkResourceTracker.getIpsPerEni();
    return offerTemplate.clone()
            .setId(offerId)
            .setSlaveId(slaveId)
            .addAllResources(asList(
                    Resource.newBuilder().setName("cpus").setType(Type.SCALAR).setScalar(Scalar.newBuilder().setValue(availableCPUs)).build(),
                    Resource.newBuilder().setName("gpu").setType(Type.SCALAR).setScalar(Scalar.newBuilder().setValue(availableGPUs)).build(),
                    Resource.newBuilder().setName("mem").setType(Type.SCALAR).setScalar(Scalar.newBuilder().setValue(availableMemory)).build(),
                    Resource.newBuilder().setName("disk").setType(Type.SCALAR).setScalar(Scalar.newBuilder().setValue(availableDisk)).build(),
                    Resource.newBuilder().setName("ports").setType(Type.RANGES).setRanges(
                            Ranges.newBuilder().addRange(Protos.Value.Range.newBuilder().setBegin(1024).setEnd(65535).build()).build()
                    ).build(),
                    Resource.newBuilder().setName("network").setType(Type.SCALAR).setScalar(Scalar.newBuilder().setValue(availableNetworkMbs)).build()
            ))
            .addAllAttributes(asList(
                    Attribute.newBuilder().setName("cluster").setType(Type.TEXT).setText(Text.newBuilder().setValue(clusterName)).build(),
                    Attribute.newBuilder().setName("asg").setType(Type.TEXT).setText(Text.newBuilder().setValue(clusterName)).build(),
                    Attribute.newBuilder().setName("id").setType(Type.TEXT).setText(Text.newBuilder().setValue(hostName)).build(),
                    Attribute.newBuilder().setName("itype").setType(Type.TEXT).setText(Text.newBuilder().setValue(instanceType.getDescriptor().getId())).build(),
                    Attribute.newBuilder().setName("SLAVE_ID").setType(Type.TEXT).setText(Text.newBuilder().setValue(clusterName)).build(),
                    Attribute.newBuilder().setName("res").setType(Type.TEXT).setText(Text.newBuilder().setValue(enis)).build()
            ))
            .build();
}
 
Example #13
Source File: SimulatedTitusAgent.java    From titus-control-plane with Apache License 2.0 6 votes vote down vote up
private void emitAvailableOffers(long delayMs) {
    synchronized (lock) {
        if (lastOffer != null) {
            rescind();
            return;
        }
        if (!shouldSendOffer()) {
            return;
        }

        if (emitSubscription != null && !emitSubscription.isUnsubscribed()) {
            emitSubscription.unsubscribe();
        }
        this.emitSubscription = worker.schedule(() -> {
            Offer newOffer = createOfferForAvailableResources();
            logger.info("Emitting new offer {}: cpu={}, memoryMB={}, networkMB={}, diskMB={}", newOffer.getId().getValue(),
                    availableCPUs, availableMemory, availableNetworkMbs, availableDisk);
            synchronized (lock) {
                this.lastOffer = newOffer;
                offerUpdates.onNext(OfferChangeEvent.offer(newOffer));
            }
        }, delayMs, TimeUnit.MILLISECONDS);
    }
}
 
Example #14
Source File: ResourceMesosScheduler.java    From oodt with Apache License 2.0 6 votes vote down vote up
/**
 * Builds a TaskInfo from the given jobspec
 * @param job - JobSpec to TaskInfo-ify
 * @param offer - offer add extra data (SlaveId)
 * @return TaskInfo fully formed
 */
private TaskInfo getTaskInfo(JobSpec job,Offer offer) {
    TaskID taskId = TaskID.newBuilder().setValue(job.getJob().getId()).build();
    TaskInfo info = TaskInfo.newBuilder().setName("task " + taskId.getValue())
             .setTaskId(taskId)
             .setSlaveId(offer.getSlaveId())
             .addResources(Resource.newBuilder()
                           .setName("cpus")
                           .setType(Value.Type.SCALAR)
                           .setScalar(Value.Scalar.newBuilder().setValue(job.getJob().getLoadValue()*1.0)))
             .addResources(Resource.newBuilder()
                           .setName("mem")
                           .setType(Value.Type.SCALAR)
                           .setScalar(Value.Scalar.newBuilder().setValue(job.getJob().getLoadValue()*1024.0)))
             .setExecutor(ExecutorInfo.newBuilder(executor)).setData(MesosUtilities.jobSpecToByteString(job)).build();
    return info;
}
 
Example #15
Source File: SimulatedTitusAgent.java    From titus-control-plane with Apache License 2.0 6 votes vote down vote up
public Observable<OfferChangeEvent> observeOffers() {
    return offerUpdates
            .compose(ObservableExt.head(() -> {
                Offer currentOffer = lastOffer;
                if (currentOffer == null) {
                    return Collections.emptyList();
                }
                return Collections.singletonList(OfferChangeEvent.offer(currentOffer));
            }))
            .distinctUntilChanged((previous, current) ->
                    previous.getOffer().getId().equals(current.getOffer().getId()) && previous.isRescind() == current.isRescind()
            )
            .doOnNext(offer -> logger.info("Sending offer update to subscribers: {}, rescinded={}", offer.getOffer().getId().getValue(), offer.isRescind()))
            .doOnSubscribe(() -> logger.info("New offer subscription for agent: {}", slaveId.getValue()))
            .doOnUnsubscribe(() -> logger.info("Offer subscription terminated for agent: {}", slaveId.getValue()));
}
 
Example #16
Source File: AttributeRuleTest.java    From dcos-commons with Apache License 2.0 6 votes vote down vote up
@Test
public void testAnyMatchesString() {
    Offer o = getOfferWithResources()
            .addAttributes(ATTR_TEXT)
            .addAttributes(ATTR_SCALAR)
            .addAttributes(ATTR_RANGES)
            .addAttributes(ATTR_SET)
            .build();
    assertTrue(AttributeRuleFactory.getInstance().require(ExactMatcher.create(AttributeStringUtils.toString(ATTR_TEXT)))
            .filter(o, POD_INSTANCE, Collections.emptyList()).isPassing());
    assertTrue(AttributeRuleFactory.getInstance().require(ExactMatcher.create(AttributeStringUtils.toString(ATTR_SCALAR)))
            .filter(o, POD_INSTANCE, Collections.emptyList()).isPassing());
    assertTrue(AttributeRuleFactory.getInstance().require(ExactMatcher.create(AttributeStringUtils.toString(ATTR_RANGES)))
            .filter(o, POD_INSTANCE, Collections.emptyList()).isPassing());
    assertTrue(AttributeRuleFactory.getInstance().require(ExactMatcher.create(AttributeStringUtils.toString(ATTR_SET)))
            .filter(o, POD_INSTANCE, Collections.emptyList()).isPassing());
}
 
Example #17
Source File: YarnNodeCapacityManagerTest.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
@Test
public void testHandleContainerAllocation() throws Exception {
  Offer offer = TestObjectFactory.getOffer("zero-localhost-one", "slave-one", "mock-framework", "offer-one", 0.1, 512.0);
  sNodeOne.allocateContainer(containerOne);
  NodeTask task = TestObjectFactory.getNodeTask("small", "localhost-one", Double.valueOf(0.1), Double.valueOf(512.0), 
      Long.parseLong("1"), Long.parseLong("256"));
  state.addNodes(Lists.newArrayList(task));
  olManager.addOffers(offer); 
  olManager.markAsConsumed(offer);
  manager.handleContainerAllocation(nodeOne);
  store.getNode("localhost-one").snapshotRunningContainers();
  assertEquals(1, store.getNode("localhost-one").getNode().getRunningContainers().size());
  assertEquals(1, store.getNode("localhost-one").getContainerSnapshot().size());
}
 
Example #18
Source File: MesosNimbus.java    From storm with Apache License 2.0 5 votes vote down vote up
public void doRegistration(final SchedulerDriver driver, Protos.FrameworkID id) {
  _driver = driver;
  // Now that we've set the driver, we can create our scheduler
  _stormScheduler = new StormSchedulerImpl(_driver);

  _state.put(FRAMEWORK_ID, id.getValue());
  _offers = new HashMap<Protos.OfferID, Protos.Offer>();

  if (_enabledLogviewerSidecar) {

    _timer.scheduleAtFixedRate(new TimerTask() {
      @Override
      public void run() {
        // performing "explicit" reconciliation; master will respond with the latest state for all logviewer tasks
        // in the framework scheduler's statusUpdate() method
        List<TaskStatus> taskStatuses = new ArrayList<TaskStatus>();
        List<String> logviewerPaths = _zkClient.getChildren(_logviewerZkDir);
        if (logviewerPaths == null) {
          _driver.reconcileTasks(taskStatuses);
          return;
        }
        for (String path : logviewerPaths) {
          TaskID logviewerTaskId = TaskID.newBuilder()
                                         .setValue(new String(_zkClient.getNodeData(String.format("%s/%s", _logviewerZkDir, path))))
                                         .build();
          TaskStatus logviewerTaskStatus = TaskStatus.newBuilder()
                                                     .setTaskId(logviewerTaskId)
                                                     .setState(TaskState.TASK_RUNNING)
                                                     .build();
          taskStatuses.add(logviewerTaskStatus);
        }
        _driver.reconcileTasks(taskStatuses);
        LOG.info("Performing task reconciliation between scheduler and master on following tasks: {}", taskStatusListToTaskIDsString(taskStatuses));
      }
    }, 0, TASK_RECONCILIATION_INTERVAL); // reconciliation performed every 5 minutes
  }
}
 
Example #19
Source File: MesosResourcePoolTest.java    From dcos-commons with Apache License 2.0 5 votes vote down vote up
@Test
public void testCreateSingleUnreservedAtomicPool() {
    Offer offer = OfferTestUtils.getOffer(ResourceTestUtils.getUnreservedMountVolume(1000, Optional.empty()));
    MesosResourcePool pool = new MesosResourcePool(offer, Optional.of(Constants.ANY_ROLE));

    Assert.assertEquals(1, pool.getUnreservedAtomicPool().size());
    Assert.assertEquals(1, pool.getUnreservedAtomicPool().get("disk").size());
}
 
Example #20
Source File: HelloWorldScheduler.java    From tutorials with MIT License 5 votes vote down vote up
@Override
public void resourceOffers(SchedulerDriver schedulerDriver, List<Offer> list) {

    for (Offer offer : list) {
        List<TaskInfo> tasks = new ArrayList<TaskInfo>();
        Protos.TaskID taskId = Protos.TaskID.newBuilder().setValue(Integer.toString(launchedTasks++)).build();

        System.out.println("Launching printHelloWorld " + taskId.getValue() + " Hello World Java");
        TaskInfo printHelloWorld = TaskInfo
                .newBuilder()
                .setName("printHelloWorld " + taskId.getValue())
                .setTaskId(taskId)
                .setSlaveId(offer.getSlaveId())
                .addResources(
                        Protos.Resource.newBuilder().setName("cpus").setType(Protos.Value.Type.SCALAR)
                                .setScalar(Protos.Value.Scalar.newBuilder().setValue(1)))
                .addResources(
                        Protos.Resource.newBuilder().setName("mem").setType(Protos.Value.Type.SCALAR)
                                .setScalar(Protos.Value.Scalar.newBuilder().setValue(128)))
                .setExecutor(ExecutorInfo.newBuilder(helloWorldExecutor)).build();

        List<OfferID> offerIDS = new ArrayList<>();
        offerIDS.add(offer.getId());

        tasks.add(printHelloWorld);

        schedulerDriver.declineOffer(offer.getId());
        schedulerDriver.launchTasks(offerIDS, tasks);
    }

}
 
Example #21
Source File: FakeMaster.java    From attic-aurora with Apache License 2.0 5 votes vote down vote up
@Override
public Status launchTasks(Collection<OfferID> offerIds, Collection<TaskInfo> tasks) {
  assertNotStopped();

  OfferID id = Iterables.getOnlyElement(offerIds);
  Offer offer = sentOffers.remove(id);
  checkState(offer != null, "Offer " + id + " is invalid.");

  final TaskInfo task = Iterables.getOnlyElement(tasks);
  synchronized (activeTasks) {
    checkState(
        !activeTasks.containsKey(task.getTaskId()),
        "Task " + task.getTaskId() + " already exists.");
    activeTasks.put(task.getTaskId(), new Task(offer, task));
  }

  executor.schedule(
      () -> Futures.getUnchecked(schedulerFuture).statusUpdate(
          this,
          TaskStatus.newBuilder()
              .setTaskId(task.getTaskId())
              .setState(TaskState.TASK_RUNNING)
              .build()),
      1,
      TimeUnit.SECONDS);

  return Status.DRIVER_RUNNING;
}
 
Example #22
Source File: NMHeartBeatHandlerTest.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
@Test 
public void testGetNewResourcesOfferedByMesos() throws Exception {
  Offer offerOne = TestObjectFactory.getOffer("localhost-one", "slave-one", "mock", "offer-one", 1.0, 512.0);
  Offer offerTwo = TestObjectFactory.getOffer("localhost-two", "slave-two", "mock", "offer-two", 2.0, 1024.0);
  olManager.addOffers(offerOne);
  olManager.addOffers(offerTwo);
  Resource resourcesOne = handler.getNewResourcesOfferedByMesos("localhost-one");
  assertEquals(1.0, resourcesOne.getVirtualCores(), 0.0);
  assertEquals(512.0, resourcesOne.getMemory(), 0.0);
  Resource resourcesTwo = handler.getNewResourcesOfferedByMesos("localhost-two");
  assertEquals(2.0, resourcesTwo.getVirtualCores(), 0.0);
  assertEquals(1024.0, resourcesTwo.getMemory(), 0.0);
}
 
Example #23
Source File: BdsMesosScheduler.java    From BigDataScript with Apache License 2.0 5 votes vote down vote up
/**
 * Match a task to the offer/s
 */
protected synchronized boolean matchTask(Task task, Host host, Collection<OfferID> offerIds, Collection<TaskInfo> taskInfos) {
	// Not enough resources in this host?
	if (!host.hasResourcesAvailable(task.getResources())) return false;

	if (verbose) Gpr.debug("Matching task: " + task.getId() + "\t resources: " + task.getResources() + "\thost:" + host.getHostName() + ", resources: " + host.getResourcesAvaialble());

	// OK, we should be able to run 'task' in hostName
	String hostName = host.toString();
	Set<Offer> offers = offersByHost.get(hostName);
	if (offers == null) {
		Gpr.debug("Offer accounting problem in host '" + hostName + "': This should ever happen!");
		cluster.remove(host); // Remove any resources since we don't really seem to have any
		return false;
	}

	// Select offers and add taskInfo
	HostResources tr = new HostResources(task.getResources());
	for (Offer offer : offers) {
		// Consume resources until offers fulfill task requirements
		HostResources or = parseOffer(offer);
		tr.consume(or);

		// Add offer and taskInfo to collections
		offerIds.add(offer.getId());
		TaskInfo taskInfo = taskInfo(offer, task);
		taskInfos.add(taskInfo);

		// Are all resources needed for this task satisfied?
		if (tr.isConsumed()) return true;
	}

	Gpr.debug("Resource accounting problem in host '" + hostName + "': This should ever happen!");
	return false;
}
 
Example #24
Source File: DefaultSchedulerTest.java    From dcos-commons with Apache License 2.0 5 votes vote down vote up
private static Protos.Offer getSufficientOfferForTaskB() {
    return Protos.Offer.newBuilder()
            .setId(Protos.OfferID.newBuilder().setValue(UUID.randomUUID().toString()).build())
            .setFrameworkId(TestConstants.FRAMEWORK_ID)
            .setSlaveId(TestConstants.AGENT_ID)
            .setHostname(TestConstants.HOSTNAME)
            .addAllResources(
                    Arrays.asList(
                            ResourceTestUtils.getUnreservedCpus(TASK_B_CPU + 0.1),
                            ResourceTestUtils.getUnreservedMem(TASK_B_MEM + 32),
                            ResourceTestUtils.getUnreservedDisk(TASK_B_DISK + 256)))
            .build();
}
 
Example #25
Source File: DefaultSchedulerTest.java    From dcos-commons with Apache License 2.0 5 votes vote down vote up
private static Protos.Offer getSufficientOfferForTaskA() {
    return Protos.Offer.newBuilder()
            .setId(Protos.OfferID.newBuilder().setValue(UUID.randomUUID().toString()).build())
            .setFrameworkId(TestConstants.FRAMEWORK_ID)
            .setSlaveId(TestConstants.AGENT_ID)
            .setHostname(TestConstants.HOSTNAME)
            .addAllResources(
                    Arrays.asList(
                            ResourceTestUtils.getUnreservedCpus(TASK_A_CPU + 0.1),
                            ResourceTestUtils.getUnreservedMem(TASK_A_MEM + 32),
                            ResourceTestUtils.getUnreservedDisk(TASK_A_DISK + 256)))
            .build();
}
 
Example #26
Source File: DefaultSchedulerTest.java    From dcos-commons with Apache License 2.0 5 votes vote down vote up
private static Protos.Offer getInsufficientOfferForTaskA(UUID offerId) {
    return Protos.Offer.newBuilder()
            .setId(Protos.OfferID.newBuilder().setValue(offerId.toString()).build())
            .setFrameworkId(TestConstants.FRAMEWORK_ID)
            .setSlaveId(TestConstants.AGENT_ID)
            .setHostname(TestConstants.HOSTNAME)
            .addAllResources(
                    Arrays.asList(
                            ResourceTestUtils.getUnreservedCpus(TASK_A_CPU / 2.0),
                            ResourceTestUtils.getUnreservedMem(TASK_A_MEM / 2.0)))
            .build();
}
 
Example #27
Source File: DefaultSchedulerTest.java    From dcos-commons with Apache License 2.0 5 votes vote down vote up
private static Protos.TaskInfo getTask(Collection<OfferRecommendation> operations) {
    for (OfferRecommendation operation : operations) {
        if (operation.getOperation().get().getType().equals(Offer.Operation.Type.LAUNCH_GROUP)) {
            return operation.getOperation().get().getLaunchGroup().getTaskGroup().getTasks(0);
        }
    }

    return null;
}
 
Example #28
Source File: DefaultSchedulerTest.java    From dcos-commons with Apache License 2.0 5 votes vote down vote up
private static List<Protos.Resource> getExpectedResources(Collection<OfferRecommendation> operations) {
    for (OfferRecommendation operation : operations) {
        if (operation.getOperation().get().getType().equals(Offer.Operation.Type.LAUNCH_GROUP)) {
            return Stream.concat(
                            operation.getOperation().get().getLaunchGroup().getTaskGroup().getTasksList().stream()
                                .flatMap(taskInfo -> taskInfo.getResourcesList().stream()),
                            operation.getOperation().get().getLaunchGroup().getExecutor().getResourcesList().stream())
                    .collect(Collectors.toList());
        }
    }

    return Collections.emptyList();
}
 
Example #29
Source File: AgentRuleTest.java    From dcos-commons with Apache License 2.0 5 votes vote down vote up
private static Offer offerWithAgent(String agentId) {
    Offer.Builder o = OfferTestUtils.getEmptyOfferBuilder();
    o.getSlaveIdBuilder().setValue(agentId);
    OfferTestUtils.addResource(o, "a");
    OfferTestUtils.addResource(o, "b");
    OfferTestUtils.addResource(o, "c");
    return o.build();
}
 
Example #30
Source File: SchedulerUtilsTest.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
@Test
public void testIsUniqueFilenameTrue() throws Exception {
  List<NodeTask> tasks = Lists.newArrayList(taskOne, taskTwo, taskThree);
  NodeTask newTask = TestObjectFactory.getNodeTask("medium", "server1", 0.4, 2048.0, Long.valueOf(1), Long.valueOf(2));
  Offer offer = TestObjectFactory.getOffer("server2", "slave1", "mock-framework", "offer1", 0.0, 0.0);
  assertTrue(SchedulerUtils.isUniqueHostname(offer, newTask, tasks));
}