Java Code Examples for org.apache.twill.api.TwillSpecification#PlacementPolicy

@Override
public TwillSpecification deserialize(JsonElement json, Type typeOfT,
                                      JsonDeserializationContext context) throws JsonParseException {
  JsonObject jsonObj = json.getAsJsonObject();

  String name = jsonObj.get("name").getAsString();
  Map<String, RuntimeSpecification> runnables = context.deserialize(
    jsonObj.get("runnables"), new TypeToken<Map<String, RuntimeSpecification>>() { }.getType());
  List<TwillSpecification.Order> orders = context.deserialize(
    jsonObj.get("orders"), new TypeToken<List<TwillSpecification.Order>>() { }.getType());
  List<TwillSpecification.PlacementPolicy> placementPolicies = context.deserialize(
    jsonObj.get("placementPolicies"), new TypeToken<List<TwillSpecification.PlacementPolicy>>() { }.getType());

  JsonElement handler = jsonObj.get("handler");
  EventHandlerSpecification eventHandler = null;
  if (handler != null && !handler.isJsonNull()) {
    eventHandler = context.deserialize(handler, EventHandlerSpecification.class);
  }

  return new DefaultTwillSpecification(name, runnables, orders, placementPolicies, eventHandler);
}
 

PlacementPolicyManager(List<TwillSpecification.PlacementPolicy> policies) {
  this.policyTypeToRunnables = new EnumMap<>(TwillSpecification.PlacementPolicy.Type.class);
  this.runnablePolicies = new HashMap<>();

  for (TwillSpecification.PlacementPolicy policy : policies) {
    policyTypeToRunnables.put(policy.getType(), policy.getNames());
    for (String runnable : policy.getNames()) {
      runnablePolicies.put(runnable, policy);
    }
  }
}
 

/**
 * Manage Blacklist for a given request.
 */
private void manageBlacklist(Map.Entry<AllocationSpecification, ? extends Collection<RuntimeSpecification>> request) {
  amClient.clearBlacklist();

  //Check the allocation strategy
  AllocationSpecification allocationSpec = request.getKey();
  if (!allocationSpec.getType().equals(AllocationSpecification.Type.ALLOCATE_ONE_INSTANCE_AT_A_TIME)) {
    return;
  }

  //Check the placement policy
  String runnableName = allocationSpec.getRunnableName();
  TwillSpecification.PlacementPolicy placementPolicy = placementPolicyManager.getPlacementPolicy(runnableName);
  if (placementPolicy == null || placementPolicy.getType() != TwillSpecification.PlacementPolicy.Type.DISTRIBUTED) {
    return;
  }

  //Update blacklist with hosts which are running DISTRIBUTED runnables
  for (String runnable : placementPolicy.getNames()) {
    for (ContainerInfo containerInfo : runningContainers.getContainerInfo(runnable)) {
      // Yarn Resource Manager may include port in the node name depending on the setting
      // YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME. It is safe to add both
      // the names (with and without port) to the blacklist.
      LOG.debug("Adding {} to host blacklist", containerInfo.getHost().getHostName());
      amClient.addToBlacklist(containerInfo.getHost().getHostName());
      amClient.addToBlacklist(containerInfo.getHost().getHostName() + ":" + containerInfo.getPort());
    }
  }
}
 

/**
 * Adds container requests with the given resource capability for each runtime.
 */
private void addContainerRequests(Resource capability,
                                  Collection<RuntimeSpecification> runtimeSpecs,
                                  Queue<ProvisionRequest> provisioning,
                                  AllocationSpecification.Type allocationType) {
  for (RuntimeSpecification runtimeSpec : runtimeSpecs) {
    String name = runtimeSpec.getName();
    int newContainers = expectedContainers.getExpected(name) - runningContainers.count(name);
    if (newContainers > 0) {
      if (allocationType.equals(AllocationSpecification.Type.ALLOCATE_ONE_INSTANCE_AT_A_TIME)) {
        //Spawning 1 instance at a time
        newContainers = 1;
      }

      // TODO: Allow user to set priority?
      LOG.info("Request {} containers with capability {} for runnable {}", newContainers, capability, name);
      YarnAMClient.ContainerRequestBuilder builder = amClient.addContainerRequest(capability, newContainers);
      builder.setPriority(0);

      TwillSpecification.PlacementPolicy placementPolicy = placementPolicyManager.getPlacementPolicy(name);
      if (placementPolicy != null) {
        builder.addHosts(placementPolicy.getHosts())
               .addRacks(placementPolicy.getRacks());
      }

      String requestId = builder.apply();
      provisioning.add(new ProvisionRequest(runtimeSpec, requestId, newContainers, allocationType));
    }
  }
}
 

@Override
public JsonElement serialize(TwillSpecification.PlacementPolicy src, Type typeOfSrc,
                             JsonSerializationContext context) {
  JsonObject json = new JsonObject();
  json.add("names", context.serialize(src.getNames(), new TypeToken<Set<String>>() { }.getType()));
  json.addProperty("type", src.getType().name());
  json.add("hosts", context.serialize(src.getHosts(), new TypeToken<Set<String>>() { }.getType()));
  json.add("racks", context.serialize(src.getRacks(), new TypeToken<Set<String>>() { }.getType()));
  return json;
}
 

@Override
public TwillSpecification.PlacementPolicy deserialize(JsonElement json, Type typeOfT,
                                                      JsonDeserializationContext context)
  throws JsonParseException {
  JsonObject jsonObj = json.getAsJsonObject();
  Set<String> names = context.deserialize(jsonObj.get("names"), new TypeToken<Set<String>>() { }.getType());
  TwillSpecification.PlacementPolicy.Type type =
    TwillSpecification.PlacementPolicy.Type.valueOf(jsonObj.get("type").getAsString());
  Set<String> hosts = context.deserialize(jsonObj.get("hosts"), new TypeToken<Set<String>>() { }.getType());
  Set<String> racks = context.deserialize(jsonObj.get("racks"), new TypeToken<Set<String>>() { }.getType());
  return new DefaultTwillSpecification.DefaultPlacementPolicy(names, type, new Hosts(hosts), new Racks(racks));
}
 

/**
 * Get the placement policy of the runnable.
 * Returns null if runnable does not belong to a placement policy.
 * @param runnableName Name of runnable.
 * @return Placement policy of the runnable.
 */
@Nullable
TwillSpecification.PlacementPolicy getPlacementPolicy(String runnableName) {
  return runnablePolicies.get(runnableName);
}