org.apache.twill.api.TwillSpecification Java Examples

@Test
public void testOrder() {
  TwillSpecification spec =
    TwillSpecification.Builder.with()
      .setName("Testing")
      .withRunnable()
      .add("r1", new DummyRunnable()).noLocalFiles()
      .add("r2", new DummyRunnable()).noLocalFiles()
      .add("r3", new DummyRunnable()).noLocalFiles()
      .add("r4", new DummyRunnable()).noLocalFiles()
      .withOrder().begin("r1", "r2").nextWhenStarted("r3")
      .build();

  Assert.assertEquals(4, spec.getRunnables().size());
  List<TwillSpecification.Order> orders = spec.getOrders();
  Assert.assertEquals(3, orders.size());
  Assert.assertEquals(ImmutableSet.of("r1", "r2"), orders.get(0).getNames());
  Assert.assertEquals(ImmutableSet.of("r3"), orders.get(1).getNames());
  Assert.assertEquals(ImmutableSet.of("r4"), orders.get(2).getNames());
}
 

@Override
public TwillSpecification configure() {
    return TwillSpecification.Builder.with()
            .setName(APPLICATION_NAME)
            .withRunnable()
                .add(PeriodicNotificationTwillRunnable.TWILL_RUNNABLE_NAME,
                        new PeriodicNotificationTwillRunnable(),
                        ResourceSpecification.Builder.with()
                            .setVirtualCores(2)
                            .setMemory(2, SizeUnit.GIGA)
                            .setInstances(1)
                            .build())
                .withLocalFiles()
                .add(PeriodicNotificationTwillRunnable.CONFIG_FILE_NAME, configFile)
                .apply()
            .anyOrder()
            .build();
}
 

@Override
public JsonElement serialize(TwillRuntimeSpecification src, Type typeOfSrc, JsonSerializationContext context) {
  JsonObject json = new JsonObject();
  json.addProperty(FS_USER, src.getFsUser());
  json.addProperty(TWILL_APP_DIR, src.getTwillAppDir().toASCIIString());
  json.addProperty(ZK_CONNECT_STR, src.getZkConnectStr());
  json.addProperty(TWILL_RUNID, src.getTwillAppRunId().getId());
  json.addProperty(TWILL_APP_NAME, src.getTwillAppName());
  if (src.getRmSchedulerAddr() != null) {
    json.addProperty(RM_SCHEDULER_ADDR, src.getRmSchedulerAddr());
  }
  json.add(TWILL_SPEC,
           context.serialize(src.getTwillSpecification(), new TypeToken<TwillSpecification>() { }.getType()));
  json.add(LOG_LEVELS,
           context.serialize(src.getLogLevels(), MAP_STRING_MAP_STRING_STRING_TYPE));
  json.add(MAX_RETRIES,
           context.serialize(src.getMaxRetries(), new TypeToken<Map<String, Integer>>() { }.getType()));
  json.add(CONFIG,
           context.serialize(src.getConfig(), new TypeToken<Map<String, String>>() { }.getType()));
  json.add(RUNNABLE_CONFIGS,
           context.serialize(src.getRunnableConfigs(), MAP_STRING_MAP_STRING_STRING_TYPE));
  return json;
}
 

private TwillRuntimeSpecificationAdapter() {
  gson = new GsonBuilder()
            .serializeNulls()
            .registerTypeAdapter(TwillRuntimeSpecification.class, new TwillRuntimeSpecificationCodec())
            .registerTypeAdapter(TwillSpecification.class, new TwillSpecificationCodec())
            .registerTypeAdapter(TwillSpecification.Order.class, new TwillSpecificationOrderCoder())
            .registerTypeAdapter(TwillSpecification.PlacementPolicy.class,
                                 new TwillSpecificationPlacementPolicyCoder())
            .registerTypeAdapter(EventHandlerSpecification.class, new EventHandlerSpecificationCoder())
            .registerTypeAdapter(RuntimeSpecification.class, new RuntimeSpecificationCodec())
            .registerTypeAdapter(TwillRunnableSpecification.class, new TwillRunnableSpecificationCodec())
            .registerTypeAdapter(ResourceSpecification.class, new ResourceSpecificationCodec())
            .registerTypeAdapter(LocalFile.class, new LocalFileCodec())
            .registerTypeAdapterFactory(new TwillSpecificationTypeAdapterFactory())
            .create();
}
 

@Override
public TwillSpecification deserialize(JsonElement json, Type typeOfT,
                                      JsonDeserializationContext context) throws JsonParseException {
  JsonObject jsonObj = json.getAsJsonObject();

  String name = jsonObj.get("name").getAsString();
  Map<String, RuntimeSpecification> runnables = context.deserialize(
    jsonObj.get("runnables"), new TypeToken<Map<String, RuntimeSpecification>>() { }.getType());
  List<TwillSpecification.Order> orders = context.deserialize(
    jsonObj.get("orders"), new TypeToken<List<TwillSpecification.Order>>() { }.getType());
  List<TwillSpecification.PlacementPolicy> placementPolicies = context.deserialize(
    jsonObj.get("placementPolicies"), new TypeToken<List<TwillSpecification.PlacementPolicy>>() { }.getType());

  JsonElement handler = jsonObj.get("handler");
  EventHandlerSpecification eventHandler = null;
  if (handler != null && !handler.isJsonNull()) {
    eventHandler = context.deserialize(handler, EventHandlerSpecification.class);
  }

  return new DefaultTwillSpecification(name, runnables, orders, placementPolicies, eventHandler);
}
 

@Override
public JsonElement serialize(TwillSpecification src, Type typeOfSrc, JsonSerializationContext context) {
  JsonObject json = new JsonObject();
  json.addProperty("name", src.getName());
  json.add("runnables", context.serialize(src.getRunnables(),
                                          new TypeToken<Map<String, RuntimeSpecification>>() { }.getType()));
  json.add("orders", context.serialize(src.getOrders(),
                                       new TypeToken<List<TwillSpecification.Order>>() { }.getType()));
  json.add("placementPolicies", context.serialize(
    src.getPlacementPolicies(), new TypeToken<List<TwillSpecification.PlacementPolicy>>() { }.getType()));
  EventHandlerSpecification eventHandler = src.getEventHandler();
  if (eventHandler != null) {
    json.add("handler", context.serialize(eventHandler, EventHandlerSpecification.class));
  }

  return json;
}
 

public TwillRuntimeSpecification(TwillSpecification twillSpecification, String fsUser, URI twillAppDir,
                                 String zkConnectStr, RunId twillRunId, String twillAppName,
                                 @Nullable String rmSchedulerAddr, Map<String, Map<String, String>> logLevels,
                                 Map<String, Integer> maxRetries, Map<String, String> config,
                                 Map<String, Map<String, String>> runnableConfigs) {
  this.twillSpecification = twillSpecification;
  this.fsUser = fsUser;
  this.twillAppDir = twillAppDir;
  this.zkConnectStr = zkConnectStr;
  this.twillRunId = twillRunId;
  this.twillAppName = twillAppName;
  this.rmSchedulerAddr = rmSchedulerAddr;
  this.logLevels = logLevels;
  this.maxRetries = maxRetries;
  this.config = config;
  this.runnableConfigs = runnableConfigs;
}
 

@Override
public TwillSpecification configure() {
  return TwillSpecification.Builder.with()
    .setName("ResourceApplication")
    .withRunnable()
      .add("echo1", new EchoServer(), ResourceSpecification.Builder.with()
        .setVirtualCores(1)
        .setMemory(256, ResourceSpecification.SizeUnit.MEGA)
        .setInstances(2).build()).noLocalFiles()
      .add("echo2", new EchoServer(), ResourceSpecification.Builder.with()
        .setVirtualCores(2)
        .setMemory(512, ResourceSpecification.SizeUnit.MEGA)
        .setInstances(1).build()).noLocalFiles()
    .anyOrder()
    .build();
}
 

@Test
public void testAnyOrder() {
  TwillSpecification spec =
    TwillSpecification.Builder.with()
      .setName("Testing")
      .withRunnable()
      .add("r1", new DummyRunnable()).noLocalFiles()
      .add("r2", new DummyRunnable()).noLocalFiles()
      .add("r3", new DummyRunnable()).noLocalFiles()
      .anyOrder()
      .build();

  Assert.assertEquals(3, spec.getRunnables().size());
  List<TwillSpecification.Order> orders = spec.getOrders();
  Assert.assertEquals(1, orders.size());
  Assert.assertEquals(ImmutableSet.of("r1", "r2", "r3"), orders.get(0).getNames());
}
 

@Override
public TwillSpecification configure() {
  // Make the runnable request for container smaller than 128MB (the allocation minimum)
  ResourceSpecification res = ResourceSpecification.Builder.with()
    .setVirtualCores(1)
    .setMemory(16, ResourceSpecification.SizeUnit.MEGA)
    .build();

  return TwillSpecification.Builder.with()
    .setName("MaxHeapApp")
    .withRunnable()
    .add("sleep", new MaxHeapRunnable(12345), res).noLocalFiles()
    .add("sleep2", new MaxHeapRunnable(23456), res).noLocalFiles()
    .anyOrder()
    .build();
}
 

@Override
public TwillSpecification configure() {
  ResourceSpecification largeRes = ResourceSpecification.Builder.with()
    .setVirtualCores(1)
    .setMemory(1024, ResourceSpecification.SizeUnit.MEGA)
    .build();

  ResourceSpecification smallRes = ResourceSpecification.Builder.with()
    .setVirtualCores(1)
    .setMemory(512, ResourceSpecification.SizeUnit.MEGA)
    .build();

  return TwillSpecification.Builder.with()
    .setName("SleepApp")
    .withRunnable()
      .add("sleep1", new SleepRunnable(12345), largeRes).noLocalFiles()
      .add("sleep2", new SleepRunnable(12346), smallRes).noLocalFiles()
    .withOrder()
      .begin("sleep1")
      .nextWhenStarted("sleep2")
    .build();
}
 

@SuppressWarnings("unchecked")
private EventHandler createEventHandler(TwillSpecification twillSpec) throws ClassNotFoundException {
  // Should be able to load by this class ClassLoader, as they packaged in the same jar.
  EventHandlerSpecification handlerSpec = twillSpec.getEventHandler();
  if (handlerSpec == null) {
    // if no handler is specified, return an EventHandler with no-op
    return new EventHandler() {};
  }

  Class<?> handlerClass = getClass().getClassLoader().loadClass(handlerSpec.getClassName());
  Preconditions.checkArgument(EventHandler.class.isAssignableFrom(handlerClass),
                              "Class {} does not implements {}",
                              handlerClass, EventHandler.class.getName());
  final EventHandler delegate = Instances.newInstance((Class<? extends EventHandler>) handlerClass);
  // wrap all calls to the delegate EventHandler methods except initialize so that all errors will be caught
  return new EventHandler() {
 

@Override
public TwillSpecification configure() {
  return TwillSpecification.Builder.with()
    .setName("SleepApplication")
    .withRunnable()
    .add(new SleepRunnable(parentFolderPath))
    .noLocalFiles()
    .anyOrder()
    .withEventHandler(new Handler(parentFolderPath))
    .build();
}
 

/**
 * Manage Blacklist for a given request.
 */
private void manageBlacklist(Map.Entry<AllocationSpecification, ? extends Collection<RuntimeSpecification>> request) {
  amClient.clearBlacklist();

  //Check the allocation strategy
  AllocationSpecification allocationSpec = request.getKey();
  if (!allocationSpec.getType().equals(AllocationSpecification.Type.ALLOCATE_ONE_INSTANCE_AT_A_TIME)) {
    return;
  }

  //Check the placement policy
  String runnableName = allocationSpec.getRunnableName();
  TwillSpecification.PlacementPolicy placementPolicy = placementPolicyManager.getPlacementPolicy(runnableName);
  if (placementPolicy == null || placementPolicy.getType() != TwillSpecification.PlacementPolicy.Type.DISTRIBUTED) {
    return;
  }

  //Update blacklist with hosts which are running DISTRIBUTED runnables
  for (String runnable : placementPolicy.getNames()) {
    for (ContainerInfo containerInfo : runningContainers.getContainerInfo(runnable)) {
      // Yarn Resource Manager may include port in the node name depending on the setting
      // YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME. It is safe to add both
      // the names (with and without port) to the blacklist.
      LOG.debug("Adding {} to host blacklist", containerInfo.getHost().getHostName());
      amClient.addToBlacklist(containerInfo.getHost().getHostName());
      amClient.addToBlacklist(containerInfo.getHost().getHostName() + ":" + containerInfo.getPort());
    }
  }
}
 

@Override
public TwillSpecification configure() {
  return TwillSpecification.Builder.with()
    .setName(JvmOptionsApplication.class.getSimpleName())
    .withRunnable()
      .add("r1", new SimpleRunnable()).noLocalFiles()
      .add("r2", new SimpleRunnable()).noLocalFiles()
      .add("r3", new SimpleRunnable()).noLocalFiles()
    .anyOrder()
    .build();
}
 

@Override
public TwillSpecification configure() {
  return TwillSpecification.Builder.with()
    .setName("EchoApp")
    .withRunnable()
      .add("echo1", new EnvironmentEchoServer()).noLocalFiles()
      .add("echo2", new EnvironmentEchoServer()).noLocalFiles()
    .anyOrder()
    .build();
}
 

@Override
public TwillSpecification configure() {
  return TwillSpecification.Builder.with()
    .setName("ServiceApp")
    .withRunnable()
      .add("server", new EchoServer()).noLocalFiles()
      .add("client", new EchoClient()).noLocalFiles()
    .anyOrder()
    .build();
}
 

@Override
public TwillSpecification configure() {
  return TwillSpecification.Builder.with()
    .setName("TestApp")
    .withRunnable()
    .add(new TestRunnable()).noLocalFiles()
    .anyOrder()
    .withEventHandler(eventHandler).build();
}
 

@Override
public TwillSpecification configure() {
  return TwillSpecification.Builder.with()
    .setName(LogLevelTestApplication.class.getSimpleName())
    .withRunnable()
    .add(LogLevelTestRunnable.class.getSimpleName(), new LogLevelTestRunnable()).noLocalFiles()
    .anyOrder()
    .build();
}
 

@Override
public TwillSpecification configure() {
  return TwillSpecification.Builder.with()
    .setName(RestartTestApplication.class.getSimpleName())
    .withRunnable()
    .add(HANGING_RUNNABLE, new HangingRunnable()).noLocalFiles()
    .add(STOPPING_RUNNABLE, new StoppingRunnable()).noLocalFiles()
    .withOrder()
    .begin(HANGING_RUNNABLE)
    .nextWhenStarted(STOPPING_RUNNABLE)
    .build();
}
 

@Override
public TwillSpecification configure() {
  return TwillSpecification.Builder.with()
    .setName(RestartTestApplication.class.getSimpleName())
    .withRunnable()
    .add(HANGING_RUNNABLE, new HangingRunnable()).noLocalFiles()
    .anyOrder()
    .build();
}
 

@Override
public TwillSpecification configure() {
  return TwillSpecification.Builder.with()
    .setName("LogLevelChangeTest")
    .withRunnable()
    .add(LogLevelTestRunnable.class.getSimpleName(), new LogLevelTestRunnable()).noLocalFiles()
    .add(LogLevelTestSecondRunnable.class.getSimpleName(), new LogLevelTestSecondRunnable()).noLocalFiles()
    .anyOrder()
    .build();
}
 

@Override
public TwillSpecification configure() {
  return TwillSpecification.Builder.with()
    .setName("TimeoutApplication")
    .withRunnable()
    .add(new TimeoutRunnable(),
         ResourceSpecification.Builder.with()
           .setVirtualCores(1)
           .setMemory(8, ResourceSpecification.SizeUnit.GIGA).build())
    .noLocalFiles()
    .anyOrder()
    .withEventHandler(new Handler(parentFolderPath))
    .build();
}
 

@Override
public TwillSpecification configure() {
  TwillRunnableSpecification runnableSpec = runnable.configure();
  return TwillSpecification.Builder.with()
    .setName(runnableSpec.getName())
    .withRunnable().add(runnableSpec.getName(), runnable, resourceSpec)
    .noLocalFiles()
    .anyOrder()
    .build();
}
 

PlacementPolicyManager(List<TwillSpecification.PlacementPolicy> policies) {
  this.policyTypeToRunnables = new EnumMap<>(TwillSpecification.PlacementPolicy.Type.class);
  this.runnablePolicies = new HashMap<>();

  for (TwillSpecification.PlacementPolicy policy : policies) {
    policyTypeToRunnables.put(policy.getType(), policy.getNames());
    for (String runnable : policy.getNames()) {
      runnablePolicies.put(runnable, policy);
    }
  }
}
 

ExpectedContainers(TwillSpecification twillSpec) {
  Map<String, ExpectedCount> expectedCounts = Maps.newHashMap();
  long now = System.currentTimeMillis();
  for (RuntimeSpecification runtimeSpec : twillSpec.getRunnables().values()) {
    expectedCounts.put(runtimeSpec.getName(),
                       new ExpectedCount(runtimeSpec.getResourceSpecification().getInstances(), now));
  }
  this.expectedCounts = expectedCounts;
}
 

@Override
public JsonElement serialize(TwillSpecification.Order src, Type typeOfSrc, JsonSerializationContext context) {
  JsonObject json = new JsonObject();
  json.add("names", context.serialize(src.getNames(), new TypeToken<Set<String>>() { }.getType()));
  json.addProperty("type", src.getType().name());
  return json;
}
 

@Override
public TwillSpecification.Order deserialize(JsonElement json, Type typeOfT,
                                            JsonDeserializationContext context) throws JsonParseException {
  JsonObject jsonObj = json.getAsJsonObject();

  Set<String> names = context.deserialize(jsonObj.get("names"), new TypeToken<Set<String>>() { }.getType());
  TwillSpecification.Order.Type type = TwillSpecification.Order.Type.valueOf(jsonObj.get("type").getAsString());

  return new DefaultTwillSpecification.DefaultOrder(names, type);
}
 

@Override
public JsonElement serialize(TwillSpecification.PlacementPolicy src, Type typeOfSrc,
                             JsonSerializationContext context) {
  JsonObject json = new JsonObject();
  json.add("names", context.serialize(src.getNames(), new TypeToken<Set<String>>() { }.getType()));
  json.addProperty("type", src.getType().name());
  json.add("hosts", context.serialize(src.getHosts(), new TypeToken<Set<String>>() { }.getType()));
  json.add("racks", context.serialize(src.getRacks(), new TypeToken<Set<String>>() { }.getType()));
  return json;
}
 

@Override
public TwillSpecification.PlacementPolicy deserialize(JsonElement json, Type typeOfT,
                                                      JsonDeserializationContext context)
  throws JsonParseException {
  JsonObject jsonObj = json.getAsJsonObject();
  Set<String> names = context.deserialize(jsonObj.get("names"), new TypeToken<Set<String>>() { }.getType());
  TwillSpecification.PlacementPolicy.Type type =
    TwillSpecification.PlacementPolicy.Type.valueOf(jsonObj.get("type").getAsString());
  Set<String> hosts = context.deserialize(jsonObj.get("hosts"), new TypeToken<Set<String>>() { }.getType());
  Set<String> racks = context.deserialize(jsonObj.get("racks"), new TypeToken<Set<String>>() { }.getType());
  return new DefaultTwillSpecification.DefaultPlacementPolicy(names, type, new Hosts(hosts), new Racks(racks));
}