org.apache.mesos.Protos.Value Java Examples

public static Value subtract(Value val1, Value val2) {
  Type type1 = val1.getType();
  Type type2 = val2.getType();

  if (type1 != type2) {
    throw new IllegalArgumentException(String.format(
        "Values to subtract do not have matching type: %s vs %s",
        TextFormat.shortDebugString(val1),
        TextFormat.shortDebugString(val2)));
  }

  switch (type1) {
    case SCALAR:
      Value.Scalar scalar = subtract(val1.getScalar(), val2.getScalar());
      return Value.newBuilder().setType(type1).setScalar(scalar).build();
    case RANGES:
      Value.Ranges ranges = subtract(val1.getRanges(), val2.getRanges());
      return Value.newBuilder().setType(type1).setRanges(ranges).build();
    default:
      throw new IllegalArgumentException(String.format(
          "Unsupported type %s when subtracting %s from %s",
          type1,
          TextFormat.shortDebugString(val2),
          TextFormat.shortDebugString(val1)));
  }
}
 

/**
 * Builds a TaskInfo from the given jobspec
 * @param job - JobSpec to TaskInfo-ify
 * @param offer - offer add extra data (SlaveId)
 * @return TaskInfo fully formed
 */
private TaskInfo getTaskInfo(JobSpec job,Offer offer) {
    TaskID taskId = TaskID.newBuilder().setValue(job.getJob().getId()).build();
    TaskInfo info = TaskInfo.newBuilder().setName("task " + taskId.getValue())
             .setTaskId(taskId)
             .setSlaveId(offer.getSlaveId())
             .addResources(Resource.newBuilder()
                           .setName("cpus")
                           .setType(Value.Type.SCALAR)
                           .setScalar(Value.Scalar.newBuilder().setValue(job.getJob().getLoadValue()*1.0)))
             .addResources(Resource.newBuilder()
                           .setName("mem")
                           .setType(Value.Type.SCALAR)
                           .setScalar(Value.Scalar.newBuilder().setValue(job.getJob().getLoadValue()*1024.0)))
             .setExecutor(ExecutorInfo.newBuilder(executor)).setData(MesosUtilities.jobSpecToByteString(job)).build();
    return info;
}
 

public static Integer compare(Value val1, Value val2) {
  Type type1 = val1.getType();
  Type type2 = val2.getType();

  if (type1 != type2) {
    throw new IllegalArgumentException(String.format(
        "Values to compare do not have matching type: %s vs %s",
        TextFormat.shortDebugString(val1),
        TextFormat.shortDebugString(val2)));
  }

  switch (type1) {
    case SCALAR:
      return compare(val1.getScalar(), val2.getScalar());
    case RANGES:
      return compare(val1.getRanges(), val2.getRanges());
    default:
      throw new IllegalArgumentException(String.format(
          "Unsupported type %s when comparing values: %s vs %s",
          type1,
          TextFormat.shortDebugString(val1),
          TextFormat.shortDebugString(val2)));
  }
}
 

public static Value add(Value val1, Value val2) {
  Type type1 = val1.getType();
  Type type2 = val2.getType();

  if (type1 != type2) {
    throw new IllegalArgumentException(String.format(
        "Values to add do not have matching type: %s vs %s",
        TextFormat.shortDebugString(val1),
        TextFormat.shortDebugString(val2)));
  }

  switch (type1) {
    case SCALAR:
      Value.Scalar scalar = add(val1.getScalar(), val2.getScalar());
      return Value.newBuilder().setType(type1).setScalar(scalar).build();
    case RANGES:
      Value.Ranges ranges = add(val1.getRanges(), val2.getRanges());
      return Value.newBuilder().setType(type1).setRanges(ranges).build();
    default:
      throw new IllegalArgumentException(String.format(
          "Unsupported type %s when adding %s to %s",
          type1,
          TextFormat.shortDebugString(val1),
          TextFormat.shortDebugString(val2)));
  }
}
 

public static Value getValue(Resource resource) {
  Type type = resource.getType();
  Value.Builder builder = Value.newBuilder();
  builder.setType(type);

  switch (type) {
    case SCALAR:
      return builder.setScalar(resource.getScalar()).build();
    case RANGES:
      return builder.setRanges(resource.getRanges()).build();
    case SET:
      return builder.setSet(resource.getSet()).build();
    default:
      throw new IllegalArgumentException(String.format("Unsupported value type %s in resource %s",
          type,
          TextFormat.shortDebugString(resource)));
  }
}
 

private Map<String, Map<String, Value>> getReservableMergedPool(
    Collection<MesosResource> mesosResources)
{
  Map<String, List<MesosResource>> rolePool = new HashMap<>();
  for (MesosResource mesosResource : getMergedResources(mesosResources)) {
    if (!mesosResource.hasResourceId()) {
      String role = mesosResource.getRole();
      List<MesosResource> resources = rolePool.get(role);
      if (resources == null) {
        resources = new ArrayList<>();
      }

      resources.add(mesosResource);
      rolePool.put(role, resources);
    }
  }

  Map<String, Map<String, Value>> roleResourcePool = new HashMap<>();
  for (Map.Entry<String, List<MesosResource>> entry : rolePool.entrySet()) {
    roleResourcePool.put(entry.getKey(), getResourcePool(entry.getValue()));
  }

  return roleResourcePool;
}
 

private void freeMergedResource(MesosResource mesosResource) {
  if (mesosResource.getResourceId().isPresent()) {
    dynamicallyReservedPoolByResourceId.remove(mesosResource.getResourceId().get());
    LOGGER.info("Freed resource: {}",
        !dynamicallyReservedPoolByResourceId.containsKey(mesosResource.getResourceId().get()));
  }

  String previousRole = mesosResource.getPreviousRole();
  Map<String, Value> pool = reservableMergedPoolByRole.get(previousRole);
  if (pool == null) {
    pool = new HashMap<>();
  }

  Value currValue = pool.get(mesosResource.getName());
  if (currValue == null) {
    currValue = ValueUtils.getZero(mesosResource.getType());
  }

  Value updatedValue = ValueUtils.add(currValue, mesosResource.getValue());
  pool.put(mesosResource.getName(), updatedValue);
  reservableMergedPoolByRole.put(previousRole, pool);
}
 

private static Map<String, Value> getResourcePool(Collection<MesosResource> mesosResources) {
  Map<String, Value> pool = new HashMap<>();
  for (MesosResource mesosResource : mesosResources) {
    String name = mesosResource.getName();
    Value currValue = pool.get(name);

    if (currValue == null) {
      currValue = ValueUtils.getZero(mesosResource.getType());
    }

    pool.put(name, ValueUtils.add(currValue, mesosResource.getValue()));
  }

  return pool;
}
 

private List<Resource> generateResources() {
  Resource rOne = Protos.Resource.newBuilder().setName("cpus").setType(Value.Type.SCALAR).setScalar(Scalar.newBuilder().setValue(0.5)).build();
  Resource rTwo = Protos.Resource.newBuilder().setName("mem").setType(Value.Type.SCALAR).setScalar(Scalar.newBuilder().setValue(1.0)).build();
  Resource rThree = Protos.Resource.newBuilder().setName("cpus").setType(Value.Type.SCALAR).setScalar(Scalar.newBuilder().setValue(1.0)).build();
  Resource rFour = Protos.Resource.newBuilder().setName("mem").setType(Value.Type.SCALAR).setScalar(Scalar.newBuilder().setValue(2.0)).build();
  
  return Lists.newArrayList(rOne, rTwo, rThree, rFour);
}
 

private static Offer offerWithAttribute(String name, String value) {
    Protos.Resource resource = ResourceBuilder.fromUnreservedValue(
            "cpus",
            Protos.Value.newBuilder()
                    .setType(Protos.Value.Type.SCALAR)
                    .setScalar(Protos.Value.Scalar.newBuilder().setValue(1.0))
                    .build())
            .build();
    Offer.Builder offerBuilder = OfferTestUtils.getCompleteOffer(resource).toBuilder();
    offerBuilder.addAttributesBuilder()
            .setName(name)
            .setType(Value.Type.TEXT)
            .getTextBuilder().setValue(value);
    return offerBuilder.build();
}
 

/**
 * Minimum to keep required field errors away.
 */
public static void addResource(Offer.Builder o, String name, String role) {
    Resource.Builder b = o.addResourcesBuilder().setType(Value.Type.RANGES).setName(name);
    if (role != null) {
        b.setRole(role);
    }
}
 

public static Value getZero(Value.Type type) {
  switch (type) {
    case SCALAR:
      Value.Scalar scalar = Value.Scalar.newBuilder().setValue(0).build();
      return Value.newBuilder().setType(type).setScalar(scalar).build();
    case RANGES:
      Value.Ranges ranges =
          Value.Ranges.newBuilder().addAllRange(Collections.emptyList()).build();
      return Value.newBuilder().setType(type).setRanges(ranges).build();
    default:
      throw new IllegalArgumentException(
          String.format("Unsupported type %s for zero value", type));
  }
}
 

private static Integer compare(Value.Ranges ranges1, Value.Ranges ranges2) {
  List<Range> list1 = ranges1.getRangeList();
  List<Range> list2 = ranges2.getRangeList();

  if (RangeUtils.rangesEqual(list1, list2)) {
    return 0;
  } else if (RangeUtils.subtractRanges(list1, list2).size() == 0) {
    return -1;
  } else {
    return 1;
  }
}
 

public Optional<MesosResource> consumeReserved(String name, Value value, String resourceId) {
  MesosResource mesosResource = dynamicallyReservedPoolByResourceId.get(resourceId);

  if (mesosResource != null) {
    if (mesosResource.isAtomic()) {
      if (sufficientValue(value, mesosResource.getValue())) {
        dynamicallyReservedPoolByResourceId.remove(resourceId);
      } else {
        LOGGER.warn("Reserved atomic quantity of {} is insufficient: desired {}, reserved {}",
            name,
            TextFormat.shortDebugString(value),
            TextFormat.shortDebugString(mesosResource.getValue()));
        return Optional.empty();
      }
    } else {
      Value availableValue = mesosResource.getValue();
      if (ValueUtils.compare(availableValue, value) > 0) {
        // Update the value in pool with the remaining unclaimed resource amount
        Resource remaining = ResourceBuilder.fromExistingResource(mesosResource.getResource())
            .setValue(ValueUtils.subtract(availableValue, value))
            .build();
        dynamicallyReservedPoolByResourceId.put(resourceId, new MesosResource(remaining));
        // Return only the claimed resource amount from this reservation
      } else {
        dynamicallyReservedPoolByResourceId.remove(resourceId);
      }
    }
  } else {
    LOGGER.warn("Failed to find reserved {} resource with ID: {}. Reserved resource IDs are: {}",
        name,
        resourceId,
        dynamicallyReservedPoolByResourceId.keySet());
  }

  return Optional.ofNullable(mesosResource);
}
 

private static boolean sufficientValue(Value desired, Value available) {
  if (desired == null) {
    return true;
  } else if (available == null) {
    return false;
  }

  Value difference = ValueUtils.subtract(desired, available);
  return ValueUtils.compare(difference, ValueUtils.getZero(desired.getType())) <= 0;
}
 

private static Integer compare(Value.Scalar scal1, Value.Scalar scal2) {
  double val1 = scal1.getValue();
  double val2 = scal2.getValue();

  if (val1 < val2) {
    return -1;
  } else if (val1 > val2) {
    return 1;
  } else {
    return 0;
  }
}
 

public static boolean equal(Value val1, Value val2) {
  return compare(val1, val2) == 0;
}
 

public Value getValue() {
  return ValueUtils.getValue(resource);
}
 

public static TaskInfo createYARNTask(Offer offer, NodeTask nodeTask) {
	NMProfile profile = nodeTask.getProfile();
	TaskID taskId = TaskID.newBuilder().setValue(nodeTask.getTaskId())
			.build();

	String taskIdValue = taskId.getValue();
	LOGGER.info("Launching task {} with profile: {}", taskIdValue, profile);
	String revisedConfig = getRevisedConfig(profile.getCpus(),
			profile.getMemory());
	String CONFIG_UPDATE_CMD = "sudo -H -u hduser bash -c \"echo '"
			+ revisedConfig
			+ "' > /usr/local/hadoop/etc/hadoop/yarn-site.xml\"; ";

	CommandInfo.Builder commandBuilder = CommandInfo.newBuilder();
	commandBuilder.setUser("root").setValue(
			CONFIG_UPDATE_CMD + NM_LAUNCH_CMD);

	TaskInfo task = TaskInfo
			.newBuilder()
			.setName("task " + taskIdValue)
			.setTaskId(taskId)
			.setSlaveId(offer.getSlaveId())
			.addResources(
					Resource.newBuilder()
							.setName("cpus")
							.setType(Value.Type.SCALAR)
							.setScalar(
									Value.Scalar.newBuilder()
											.setValue(profile.getCpus())
											.build()).build())
			.addResources(
					Resource.newBuilder()
							.setName("mem")
							.setType(Value.Type.SCALAR)
							.setScalar(
									Value.Scalar.newBuilder()
											.setValue(profile.getMemory())
											.build()).build())
			.setCommand(commandBuilder.build()).build();

	return task;
}
 

private boolean matches(Offer offer, NMProfile profile) {
	double cpus = -1;
	double mem = -1;

	for (Resource resource : offer.getResourcesList()) {
		if (resource.getName().equals("cpus")) {
			if (resource.getType().equals(Value.Type.SCALAR)) {
				cpus = resource.getScalar().getValue();
			} else {
				LOGGER.error("Cpus resource was not a scalar: {}", resource
						.getType().toString());
			}
		} else if (resource.getName().equals("mem")) {
			if (resource.getType().equals(Value.Type.SCALAR)) {
				mem = resource.getScalar().getValue();
			} else {
				LOGGER.error("Mem resource was not a scalar: {}", resource
						.getType().toString());
			}
		} else if (resource.getName().equals("disk")) {
			LOGGER.warn("Ignoring disk resources from offer");
		} else if (resource.getName().equals("ports")) {
			LOGGER.info("Ignoring ports resources from offer");
		} else {
			LOGGER.warn("Ignoring unknown resource type: {}",
					resource.getName());
		}
	}

	if (cpus < 0)
		LOGGER.error("No cpus resource present");
	if (mem < 0)
		LOGGER.error("No mem resource present");

	Map<String, String> requestAttributes = new HashMap<String, String>();

	if (profile.getCpus() <= cpus
			&& profile.getMemory() <= mem
			&& SchedulerUtils.isMatchSlaveAttributes(offer,
					requestAttributes)) {
		return true;
	} else {
		LOGGER.info("Offer not sufficient for profile: " + profile);
		return false;
	}
}
 

/**
 * Greedily acquire resources by launching a Mesos Task(w/ our custom MesosExecutor) on REEF Evaluator request.
 * Either called from onResourceRequest(for a new request) or resourceOffers(for an outstanding request).
 * TODO[JIRA REEF-102]: reflect priority and rack/node locality specified in resourceRequestEvent.
 */
private synchronized void doResourceRequest(final ResourceRequestEvent resourceRequestEvent) {
  int tasksToLaunchCounter = resourceRequestEvent.getResourceCount();

  for (final Offer offer : this.offers.values()) {
    final int cpuSlots = getCpu(offer) / resourceRequestEvent.getVirtualCores().get();
    final int memSlots = getMemory(offer) / resourceRequestEvent.getMemorySize().get();
    final int taskNum = Math.min(Math.min(cpuSlots, memSlots), tasksToLaunchCounter);

    if (taskNum > 0 && satisfySlaveConstraint(resourceRequestEvent, offer)) {
      final List<TaskInfo> tasksToLaunch = new ArrayList<>();
      tasksToLaunchCounter -= taskNum;

      // Launch as many MesosTasks on the same node(offer) as possible to exploit locality.
      for (int j = 0; j < taskNum; j++) {
        final String id = offer.getId().getValue() + "-" + String.valueOf(j);
        final String executorLaunchCommand = getExecutorLaunchCommand(id, resourceRequestEvent.getMemorySize().get());

        final ExecutorInfo executorInfo = ExecutorInfo.newBuilder()
            .setExecutorId(ExecutorID.newBuilder()
                .setValue(id)
                .build())
            .setCommand(CommandInfo.newBuilder()
                .setValue(executorLaunchCommand)
                .addUris(URI.newBuilder().setValue(reefTarUri).build())
                .build())
            .build();

        final TaskInfo taskInfo = TaskInfo.newBuilder()
            .setTaskId(TaskID.newBuilder()
                .setValue(id)
                .build())
            .setName(id)
            .setSlaveId(offer.getSlaveId())
            .addResources(Resource.newBuilder()
                    .setName("mem")
                    .setType(Type.SCALAR)
                    .setScalar(Value.Scalar.newBuilder()
                            .setValue(resourceRequestEvent.getMemorySize().get())
                            .build())
                    .build())
            .addResources(Resource.newBuilder()
                    .setName("cpus")
                    .setType(Type.SCALAR)
                    .setScalar(Value.Scalar.newBuilder()
                            .setValue(resourceRequestEvent.getVirtualCores().get())
                            .build())
                    .build())
            .setExecutor(executorInfo)
            .build();

        tasksToLaunch.add(taskInfo);
        this.executorIdToLaunchedRequests.put(id, resourceRequestEvent);
      }

      final Filters filters = Filters.newBuilder().setRefuseSeconds(0).build();
      mesosMaster.launchTasks(Collections.singleton(offer.getId()), tasksToLaunch, filters);
    } else {
      mesosMaster.declineOffer(offer.getId());
    }
  }

  // the offers are no longer valid(all launched or declined)
  this.offers.clear();

  // Save leftovers that couldn't be launched
  outstandingRequests.add(ResourceRequestEventImpl.newBuilder()
      .mergeFrom(resourceRequestEvent)
      .setResourceCount(tasksToLaunchCounter)
      .build());
}
 

private static Value.Ranges subtract(Value.Ranges ranges1, Value.Ranges ranges2) {
  List<Range> list1 = ranges1.getRangeList();
  List<Range> list2 = ranges2.getRangeList();
  List<Range> list = RangeUtils.subtractRanges(list1, list2);
  return Value.Ranges.newBuilder().addAllRange(list).build();
}
 

private static Value.Scalar subtract(Value.Scalar scal1, Value.Scalar scal2) {
  double value = scal1.getValue() - scal2.getValue();
  return Value.Scalar.newBuilder().setValue(value).build();
}
 

public Value.Type getType() {
  return resource.getType();
}
 

private static Value.Ranges add(Value.Ranges ranges1, Value.Ranges ranges2) {
  List<Range> list1 = ranges1.getRangeList();
  List<Range> list2 = ranges2.getRangeList();
  List<Range> list = RangeUtils.mergeRanges(list1, list2);
  return Value.Ranges.newBuilder().addAllRange(list).build();
}
 

private static Value.Scalar add(Value.Scalar scal1, Value.Scalar scal2) {
  double value = scal1.getValue() + scal2.getValue();
  return Value.Scalar.newBuilder().setValue(value).build();
}
 

public Map<String, Value> getUnreservedMergedPool() {
  Map<String, Value> pool = reservableMergedPoolByRole.get(Constants.ANY_ROLE);
  return pool == null ? Collections.emptyMap() : pool;
}
 

public Map<String, Value> getUnreservedMergedPoolByRole(String preReservedRole) {
  Map<String, Value> pool = reservableMergedPoolByRole.get(preReservedRole);
  return pool == null ? Collections.emptyMap() : pool;
}
 

public Optional<MesosResource> consumeReservableMerged(
    String name,
    Value desiredValue,
    String preReservedRole)
{
  Map<String, Value> pool = reservableMergedPoolByRole.get(preReservedRole);
  if (pool == null) {
    LOGGER.info("No unreserved resources available for role '{}'. Reservable roles are: {}",
        preReservedRole, reservableMergedPoolByRole.keySet());
    return Optional.empty();
  }

  Value availableValue = pool.get(name);

  if (sufficientValue(desiredValue, availableValue)) {
    pool.put(name, ValueUtils.subtract(availableValue, desiredValue));
    reservableMergedPoolByRole.put(preReservedRole, pool);

    Resource.Builder builder =
        ResourceBuilder.fromUnreservedValue(name, desiredValue).build().toBuilder();
    if (Capabilities.getInstance().supportsPreReservedResources() &&
        !preReservedRole.equals(Constants.ANY_ROLE))
    {
      builder.addReservations(
          Resource.ReservationInfo.newBuilder()
              .setRole(preReservedRole)
              .setType(Resource.ReservationInfo.Type.STATIC));
    }

    return Optional.of(new MesosResource(builder.build()));
  } else {
    if (availableValue == null) {
      LOGGER.info("Offer lacks any unreserved {} resources for role {}", name, preReservedRole);
    } else {
      LOGGER.info("Offered quantity of {} for role {} is insufficient: desired {}, offered {}",
          name,
          preReservedRole,
          TextFormat.shortDebugString(desiredValue),
          TextFormat.shortDebugString(availableValue));
    }
    return Optional.empty();
  }
}
 

/**
 * Returns the resources which are reservable.  These may have been pre-reserved (dynamically or statically) or
 * never reserved.
 */
public Map<String, Map<String, Value>> getReservableMergedPoolByRole() {
  return reservableMergedPoolByRole;
}