org.apache.mesos.Protos.Value.Type Java Examples

private Offer createOfferForAvailableResources() {
    Protos.OfferID offerId = Protos.OfferID.newBuilder().setValue(slaveId.getValue() + "_O_" + offerIdx++).build();
    String enis = "ResourceSet-ENIs-7-" + networkResourceTracker.getIpsPerEni();
    return offerTemplate.clone()
            .setId(offerId)
            .setSlaveId(slaveId)
            .addAllResources(asList(
                    Resource.newBuilder().setName("cpus").setType(Type.SCALAR).setScalar(Scalar.newBuilder().setValue(availableCPUs)).build(),
                    Resource.newBuilder().setName("gpu").setType(Type.SCALAR).setScalar(Scalar.newBuilder().setValue(availableGPUs)).build(),
                    Resource.newBuilder().setName("mem").setType(Type.SCALAR).setScalar(Scalar.newBuilder().setValue(availableMemory)).build(),
                    Resource.newBuilder().setName("disk").setType(Type.SCALAR).setScalar(Scalar.newBuilder().setValue(availableDisk)).build(),
                    Resource.newBuilder().setName("ports").setType(Type.RANGES).setRanges(
                            Ranges.newBuilder().addRange(Protos.Value.Range.newBuilder().setBegin(1024).setEnd(65535).build()).build()
                    ).build(),
                    Resource.newBuilder().setName("network").setType(Type.SCALAR).setScalar(Scalar.newBuilder().setValue(availableNetworkMbs)).build()
            ))
            .addAllAttributes(asList(
                    Attribute.newBuilder().setName("cluster").setType(Type.TEXT).setText(Text.newBuilder().setValue(clusterName)).build(),
                    Attribute.newBuilder().setName("asg").setType(Type.TEXT).setText(Text.newBuilder().setValue(clusterName)).build(),
                    Attribute.newBuilder().setName("id").setType(Type.TEXT).setText(Text.newBuilder().setValue(hostName)).build(),
                    Attribute.newBuilder().setName("itype").setType(Type.TEXT).setText(Text.newBuilder().setValue(instanceType.getDescriptor().getId())).build(),
                    Attribute.newBuilder().setName("SLAVE_ID").setType(Type.TEXT).setText(Text.newBuilder().setValue(clusterName)).build(),
                    Attribute.newBuilder().setName("res").setType(Type.TEXT).setText(Text.newBuilder().setValue(enis)).build()
            ))
            .build();
}
 

private SimulatedTitusAgent createAgent() {
    String hostName = ComputeResources.asHostname(computeResources.allocateIpAddress(), name);
    String zoneId = ZONES[(int) (zoneCounter.getAndIncrement() % ZONES.length)];
    Protos.SlaveID slaveId = Protos.SlaveID.newBuilder().setValue(hostName).build();

    Protos.Offer.Builder agentOfferTemplate = offerTemplate.clone()
            .setHostname(hostName)
            .addAttributes(Attribute.newBuilder().setName("zone").setType(Type.TEXT).setText(Text.newBuilder().setValue(zoneId).build()).build())
            .setUrl(Protos.URL.newBuilder()
                    .setScheme("http")
                    .setAddress(Protos.Address.newBuilder().setHostname(hostName).setPort(5051))
                    .setPath("slave")
            );

    SimulatedTitusAgent agent = new SimulatedTitusAgent(name, computeResources, hostName, slaveId, agentOfferTemplate, instanceType,
            cpus, gpus, memory, disk, networkMbs, ipPerEni, containerPlayersManager, Schedulers.computation());
    agents.put(agent.getId(), agent);
    topologUpdateSubject.onNext(AgentChangeEvent.newInstance(this, agent));
    return agent;
}
 

public static Value getValue(Resource resource) {
  Type type = resource.getType();
  Value.Builder builder = Value.newBuilder();
  builder.setType(type);

  switch (type) {
    case SCALAR:
      return builder.setScalar(resource.getScalar()).build();
    case RANGES:
      return builder.setRanges(resource.getRanges()).build();
    case SET:
      return builder.setSet(resource.getSet()).build();
    default:
      throw new IllegalArgumentException(String.format("Unsupported value type %s in resource %s",
          type,
          TextFormat.shortDebugString(resource)));
  }
}
 

public static Value add(Value val1, Value val2) {
  Type type1 = val1.getType();
  Type type2 = val2.getType();

  if (type1 != type2) {
    throw new IllegalArgumentException(String.format(
        "Values to add do not have matching type: %s vs %s",
        TextFormat.shortDebugString(val1),
        TextFormat.shortDebugString(val2)));
  }

  switch (type1) {
    case SCALAR:
      Value.Scalar scalar = add(val1.getScalar(), val2.getScalar());
      return Value.newBuilder().setType(type1).setScalar(scalar).build();
    case RANGES:
      Value.Ranges ranges = add(val1.getRanges(), val2.getRanges());
      return Value.newBuilder().setType(type1).setRanges(ranges).build();
    default:
      throw new IllegalArgumentException(String.format(
          "Unsupported type %s when adding %s to %s",
          type1,
          TextFormat.shortDebugString(val1),
          TextFormat.shortDebugString(val2)));
  }
}
 

public static Value subtract(Value val1, Value val2) {
  Type type1 = val1.getType();
  Type type2 = val2.getType();

  if (type1 != type2) {
    throw new IllegalArgumentException(String.format(
        "Values to subtract do not have matching type: %s vs %s",
        TextFormat.shortDebugString(val1),
        TextFormat.shortDebugString(val2)));
  }

  switch (type1) {
    case SCALAR:
      Value.Scalar scalar = subtract(val1.getScalar(), val2.getScalar());
      return Value.newBuilder().setType(type1).setScalar(scalar).build();
    case RANGES:
      Value.Ranges ranges = subtract(val1.getRanges(), val2.getRanges());
      return Value.newBuilder().setType(type1).setRanges(ranges).build();
    default:
      throw new IllegalArgumentException(String.format(
          "Unsupported type %s when subtracting %s from %s",
          type1,
          TextFormat.shortDebugString(val2),
          TextFormat.shortDebugString(val1)));
  }
}
 

public static Integer compare(Value val1, Value val2) {
  Type type1 = val1.getType();
  Type type2 = val2.getType();

  if (type1 != type2) {
    throw new IllegalArgumentException(String.format(
        "Values to compare do not have matching type: %s vs %s",
        TextFormat.shortDebugString(val1),
        TextFormat.shortDebugString(val2)));
  }

  switch (type1) {
    case SCALAR:
      return compare(val1.getScalar(), val2.getScalar());
    case RANGES:
      return compare(val1.getRanges(), val2.getRanges());
    default:
      throw new IllegalArgumentException(String.format(
          "Unsupported type %s when comparing values: %s vs %s",
          type1,
          TextFormat.shortDebugString(val1),
          TextFormat.shortDebugString(val2)));
  }
}
 

@Test
public void testSetAttributeString() {
    Attribute.Builder attr = Attribute.newBuilder().setType(Type.SET).setName("ram");
    attr.getSetBuilder().addItem("");
    assertEquals("ram:{}", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SET).setName("ports");
    attr.getSetBuilder().addItem("a").addItem("b").addItem("c");
    assertEquals("ports:{a,b,c}", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SET).setName("disk");
    attr.getSetBuilder().addItem("-1").addItem("-2");
    assertEquals("disk:{-1,-2}", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SET).setName("");
    attr.getSetBuilder().addItem("-1").addItem("-2");
    assertEquals(":{-1,-2}", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SET).setName("empty");
    assertEquals("empty:{}", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SET).setName("");
    assertEquals(":{}", AttributeStringUtils.toString(attr.build()));
}
 

@Test
public void testTextAttributeString() {
    Attribute.Builder attr = Attribute.newBuilder().setType(Type.TEXT).setName("ram");
    attr.getTextBuilder().setValue(":::");
    assertEquals("ram::::", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.TEXT).setName("ram");
    attr.getTextBuilder().setValue("abc");
    assertEquals("ram:abc", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.TEXT).setName("");
    attr.getTextBuilder().setValue("123");
    assertEquals(":" + "123" /* workaround for lint thinking this is an IP */,
            AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.TEXT).setName("empty");
    assertEquals("empty:", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.TEXT).setName("");
    assertEquals(":", AttributeStringUtils.toString(attr.build()));
}
 

@Test
public void testMixedAttributeStrings() {
    List<Attribute> attrs = new ArrayList<>();
    assertEquals("", AttributeStringUtils.toString(attrs));

    Attribute.Builder attr = Attribute.newBuilder().setType(Type.TEXT).setName("ram");
    attr.getTextBuilder().setValue("");
    attrs.add(attr.build());
    assertEquals("ram:", AttributeStringUtils.toString(attrs));

    attr = Attribute.newBuilder().setType(Type.SET).setName("ports");
    attr.getSetBuilder().addItem("a").addItem("b").addItem("c");
    attrs.add(attr.build());
    assertEquals("ram:;ports:{a,b,c}", AttributeStringUtils.toString(attrs));

    attr = Attribute.newBuilder().setType(Type.SCALAR).setName("roundup1");
    attr.getScalarBuilder().setValue(1.99999);
    attrs.add(attr.build());
    assertEquals("ram:;ports:{a,b,c};roundup1:2.000", AttributeStringUtils.toString(attrs));

    attr = Attribute.newBuilder().setType(Type.RANGES).setName("disk");
    attr.getRangesBuilder().addRangeBuilder().setBegin(-321).setEnd(-123);
    attrs.add(attr.build());
    assertEquals("ram:;ports:{a,b,c};roundup1:2.000;disk:[-321--123]", AttributeStringUtils.toString(attrs));
}
 

/**
 * Creates string out an offer in a nice format.
 * @param resource - mesos resource to make into string.
 * @return string representing a resource.
 */
public static String getResourceMessage(Resource resource) {
    Type type = resource.getType();
    String ret = resource.getName() +" "+resource.getRole()+ ": ";
    switch (type) {
        case SCALAR:
            ret += resource.getScalar().getValue();
            break;
        case RANGES:
            for (Range range : resource.getRanges().getRangeList())
                ret += range.getBegin() + " - "+range.getEnd()+",";
            break;
        case TEXT:
            ret += " TEXT type...cannot find.";
            break;
        case SET:
            for (String string : resource.getSet().getItemList())
                ret += string + ",";
            break;
    }
    return ret;
}
 

private SimulatedTitusAgentCluster(String name,
                                   ComputeResources computeResources,
                                   AwsInstanceType instanceType,
                                   double cpus,
                                   double gpus,
                                   int memory,
                                   int disk,
                                   int networkMbs,
                                   int ipPerEni,
                                   ContainerPlayersManager containerPlayersManager,
                                   int maxSize) {
    this.name = name;
    this.computeResources = computeResources;
    this.instanceType = instanceType;
    this.cpus = cpus;
    this.gpus = gpus;
    this.memory = memory;
    this.disk = disk;
    this.networkMbs = networkMbs;
    this.ipPerEni = ipPerEni;
    this.containerPlayersManager = containerPlayersManager;
    this.maxSize = maxSize;

    this.offerTemplate = Protos.Offer.newBuilder()
            .setFrameworkId(Protos.FrameworkID.newBuilder().setValue("EmbeddedTitusMaster"))
            .addAllAttributes(asList(
                    Attribute.newBuilder().setName("CLUSTER_NAME").setType(Type.SCALAR).setText(Text.newBuilder().setValue(name)).build(),
                    Attribute.newBuilder().setName("region").setType(Type.SCALAR).setText(Text.newBuilder().setValue("us-east-1")).build(),
                    Attribute.newBuilder().setName("stack").setType(Type.SCALAR).setText(Text.newBuilder().setValue("local")).build(),
                    Attribute.newBuilder().setName("itype").setType(Type.TEXT).setText(Text.newBuilder().setValue(instanceType.name())).build()
            ));
}
 

public static Value getZero(Value.Type type) {
  switch (type) {
    case SCALAR:
      Value.Scalar scalar = Value.Scalar.newBuilder().setValue(0).build();
      return Value.newBuilder().setType(type).setScalar(scalar).build();
    case RANGES:
      Value.Ranges ranges =
          Value.Ranges.newBuilder().addAllRange(Collections.emptyList()).build();
      return Value.newBuilder().setType(type).setRanges(ranges).build();
    default:
      throw new IllegalArgumentException(
          String.format("Unsupported type %s for zero value", type));
  }
}
 

@Test
public void testRangeAttributeString() {
    Attribute.Builder attr = Attribute.newBuilder().setType(Type.RANGES).setName("ram");
    attr.getRangesBuilder().addRangeBuilder().setBegin(1).setEnd(2);
    assertEquals("ram:[1-2]", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.RANGES).setName("ports");
    attr.getRangesBuilder().addRangeBuilder().setBegin(1).setEnd(2);
    attr.getRangesBuilder().addRangeBuilder().setBegin(-321).setEnd(-123);
    attr.getRangesBuilder().addRangeBuilder().setBegin(21000).setEnd(24000);
    attr.getRangesBuilder().addRangeBuilder().setBegin(0).setEnd(0);
    attr.getRangesBuilder().addRangeBuilder().setBegin(30000).setEnd(34000);
    attr.getRangesBuilder().addRangeBuilder().setBegin(321).setEnd(123);
    assertEquals("ports:[1-2,-321--123,21000-24000,0-0,30000-34000,321-123]",
            AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.RANGES).setName("ram");
    attr.getRangesBuilder().addRangeBuilder().setBegin(0).setEnd(0);
    assertEquals("ram:[0-0]", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.RANGES).setName("disk");
    attr.getRangesBuilder().addRangeBuilder().setBegin(-321).setEnd(-123);
    assertEquals("disk:[-321--123]", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.RANGES).setName("");
    attr.getRangesBuilder().addRangeBuilder().setBegin(-321).setEnd(-123);
    assertEquals(":[-321--123]", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.RANGES).setName("empty");
    assertEquals("empty:[]", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.RANGES).setName("");
    assertEquals(":[]", AttributeStringUtils.toString(attr.build()));
}
 

@Test
public void testScalarAttributeString() {
    Attribute.Builder attr = Attribute.newBuilder().setType(Type.SCALAR).setName("ram");
    attr.getScalarBuilder().setValue(0);
    assertEquals("ram:0.000", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SCALAR).setName("ports");
    attr.getScalarBuilder().setValue(0.000);
    assertEquals("ports:0.000", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SCALAR).setName("ports");
    attr.getScalarBuilder().setValue(0.0001);
    assertEquals("ports:0.000", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SCALAR).setName("ports");
    attr.getScalarBuilder().setValue(0.0005);
    assertEquals("ports:0.001", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SCALAR).setName("rounddown");
    attr.getScalarBuilder().setValue(-1.99999);
    assertEquals("rounddown:-2.000", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SCALAR).setName("roundup1");
    attr.getScalarBuilder().setValue(1.99999);
    assertEquals("roundup1:2.000", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SCALAR).setName("");
    attr.getScalarBuilder().setValue(1.99999);
    assertEquals(":2.000", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SCALAR).setName("roundup2");
    attr.getScalarBuilder().setValue(999999.99999);
    assertEquals("roundup2:1000000.000", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SCALAR).setName("empty");
    assertEquals("empty:0.000", AttributeStringUtils.toString(attr.build()));

    attr = Attribute.newBuilder().setType(Type.SCALAR).setName("");
    assertEquals(":0.000", AttributeStringUtils.toString(attr.build()));
}
 

public static Offer getOffer(String host, String slaveId, String frameworkId, String offerId, double cpuCores, double memory) {
  Protos.SlaveID sid = SlaveID.newBuilder().setValue(slaveId).build();
  Protos.FrameworkID fid = FrameworkID.newBuilder().setValue(frameworkId).build();
  Protos.Value.Scalar cores = Protos.Value.Scalar.newBuilder().setValue(cpuCores).build();
  Protos.Value.Scalar mem = Protos.Value.Scalar.newBuilder().setValue(memory).build();
  Protos.Resource cpuResource = Protos.Resource.newBuilder().setName("cpus").setScalar(cores).setType(Type.SCALAR).build();
  Protos.Resource memResource  = Protos.Resource.newBuilder().setName("mem").setScalar(mem).setType(Type.SCALAR).build();
  return Protos.Offer.newBuilder().setHostname(host).setId(OfferID.newBuilder().setValue(offerId)).
      setSlaveId(sid).setFrameworkId(fid).addResources(cpuResource).addResources(memResource).build();  
}
 

/**
 * Greedily acquire resources by launching a Mesos Task(w/ our custom MesosExecutor) on REEF Evaluator request.
 * Either called from onResourceRequest(for a new request) or resourceOffers(for an outstanding request).
 * TODO[JIRA REEF-102]: reflect priority and rack/node locality specified in resourceRequestEvent.
 */
private synchronized void doResourceRequest(final ResourceRequestEvent resourceRequestEvent) {
  int tasksToLaunchCounter = resourceRequestEvent.getResourceCount();

  for (final Offer offer : this.offers.values()) {
    final int cpuSlots = getCpu(offer) / resourceRequestEvent.getVirtualCores().get();
    final int memSlots = getMemory(offer) / resourceRequestEvent.getMemorySize().get();
    final int taskNum = Math.min(Math.min(cpuSlots, memSlots), tasksToLaunchCounter);

    if (taskNum > 0 && satisfySlaveConstraint(resourceRequestEvent, offer)) {
      final List<TaskInfo> tasksToLaunch = new ArrayList<>();
      tasksToLaunchCounter -= taskNum;

      // Launch as many MesosTasks on the same node(offer) as possible to exploit locality.
      for (int j = 0; j < taskNum; j++) {
        final String id = offer.getId().getValue() + "-" + String.valueOf(j);
        final String executorLaunchCommand = getExecutorLaunchCommand(id, resourceRequestEvent.getMemorySize().get());

        final ExecutorInfo executorInfo = ExecutorInfo.newBuilder()
            .setExecutorId(ExecutorID.newBuilder()
                .setValue(id)
                .build())
            .setCommand(CommandInfo.newBuilder()
                .setValue(executorLaunchCommand)
                .addUris(URI.newBuilder().setValue(reefTarUri).build())
                .build())
            .build();

        final TaskInfo taskInfo = TaskInfo.newBuilder()
            .setTaskId(TaskID.newBuilder()
                .setValue(id)
                .build())
            .setName(id)
            .setSlaveId(offer.getSlaveId())
            .addResources(Resource.newBuilder()
                    .setName("mem")
                    .setType(Type.SCALAR)
                    .setScalar(Value.Scalar.newBuilder()
                            .setValue(resourceRequestEvent.getMemorySize().get())
                            .build())
                    .build())
            .addResources(Resource.newBuilder()
                    .setName("cpus")
                    .setType(Type.SCALAR)
                    .setScalar(Value.Scalar.newBuilder()
                            .setValue(resourceRequestEvent.getVirtualCores().get())
                            .build())
                    .build())
            .setExecutor(executorInfo)
            .build();

        tasksToLaunch.add(taskInfo);
        this.executorIdToLaunchedRequests.put(id, resourceRequestEvent);
      }

      final Filters filters = Filters.newBuilder().setRefuseSeconds(0).build();
      mesosMaster.launchTasks(Collections.singleton(offer.getId()), tasksToLaunch, filters);
    } else {
      mesosMaster.declineOffer(offer.getId());
    }
  }

  // the offers are no longer valid(all launched or declined)
  this.offers.clear();

  // Save leftovers that couldn't be launched
  outstandingRequests.add(ResourceRequestEventImpl.newBuilder()
      .mergeFrom(resourceRequestEvent)
      .setResourceCount(tasksToLaunchCounter)
      .build());
}