com.datatorrent.api.Context.DAGContext Java Examples

private void prepareDAGAttributes(LogicalPlan dag)
{
  // Consider making all attributes available for DAG construction
  // EVENTUALLY to be replaced by variable enabled configuration in the demo where the attribute below is used
  String connectAddress = conf.get(KEY_GATEWAY_CONNECT_ADDRESS);
  dag.setAttribute(DAGContext.GATEWAY_CONNECT_ADDRESS, connectAddress == null ? conf.get(GATEWAY_LISTEN_ADDRESS) : connectAddress);
  if (conf.getBoolean(KEY_GATEWAY_USE_SSL, DAGContext.GATEWAY_USE_SSL.defaultValue)) {
    dag.setAttribute(DAGContext.GATEWAY_USE_SSL, true);
  }
  String username = conf.get(KEY_GATEWAY_USER_NAME);
  if (username != null) {
    dag.setAttribute(DAGContext.GATEWAY_USER_NAME, username);
  }
  String password = conf.get(KEY_GATEWAY_PASSWORD);
  if (password != null) {
    dag.setAttribute(DAGContext.GATEWAY_PASSWORD, password);
  }
}
 

@Test
public void testAppNameAttribute()
{
  StreamingApplication app = new AnnotatedApplication();
  Configuration conf = new Configuration(false);
  conf.addResource(StramClientUtils.DT_SITE_XML_FILE);

  LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);

  Properties properties = new Properties();
  properties.put(StreamingApplication.APEX_PREFIX + "application.TestAliasApp.class", app.getClass().getName());

  builder.addFromProperties(properties, null);

  LogicalPlan dag = new LogicalPlan();
  String appPath = app.getClass().getName().replace(".", "/") + ".class";
  dag.setAttribute(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME, "testApp");
  builder.prepareDAG(dag, app, appPath);

  Assert.assertEquals("Application name", "testApp", dag.getAttributes().get(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME));
}
 

@Test
public void testAppAlias()
{
  StreamingApplication app = new AnnotatedApplication();
  Configuration conf = new Configuration(false);
  conf.addResource(StramClientUtils.DT_SITE_XML_FILE);

  LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);

  Properties properties = new Properties();
  properties.put(StreamingApplication.APEX_PREFIX + "application.TestAliasApp.class", app.getClass().getName());

  builder.addFromProperties(properties, null);

  LogicalPlan dag = new LogicalPlan();
  String appPath = app.getClass().getName().replace(".", "/") + ".class";
  builder.prepareDAG(dag, app, appPath);

  Assert.assertEquals("Application name", "TestAliasApp", dag.getAttributes().get(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME));
}
 

@Test
public void testOiOCommitted() throws IOException, ClassNotFoundException
{
  LogicalPlan lp = new LogicalPlan();
  String workingDir = new File("target/testCommitted").getAbsolutePath();
  lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
  lp.setAttribute(DAGContext.CHECKPOINT_WINDOW_COUNT, 1);
  String op1Name = "CommitAwareOperatorTestOioCommit1";
  String op2Name = "CommitAwareOperatorTestOioCommit2";
  CommitAwareOperator operator1 = lp.addOperator(op1Name, new CommitAwareOperator());
  CommitAwareOperator operator2 = lp.addOperator(op2Name, new CommitAwareOperator());
  lp.addStream("local", operator1.output, operator2.input).setLocality(Locality.THREAD_LOCAL);

  StramLocalCluster lc = new StramLocalCluster(lp);
  lc.run(5000);

  /* this is not foolproof but some insurance is better than nothing */
  Assert.assertTrue("No Committed Windows", committedWindowIds.contains(op1Name));
  Assert.assertTrue("No Committed Windows", committedWindowIds.contains(op2Name));
}
 

@Test
public void testCommitted() throws IOException, ClassNotFoundException
{
  LogicalPlan lp = new LogicalPlan();
  String workingDir = new File("target/testCommitted").getAbsolutePath();
  lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
  lp.setAttribute(DAGContext.CHECKPOINT_WINDOW_COUNT, 1);
  String opName = "CommitAwareOperatorTestCommit";
  lp.addOperator(opName, new CommitAwareOperator());

  StramLocalCluster lc = new StramLocalCluster(lp);
  lc.run(5000);

  /* this is not foolproof but some insurance is better than nothing */
  Assert.assertTrue("No Committed Windows", committedWindowIds.contains(opName));
}
 

@Test
public void testAppPath() throws Exception
{
  // add operator for initial checkpoint
  TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
  o1.setMaxTuples(1);
  File relPath = new File(dag.getAttributes().get(DAGContext.APPLICATION_PATH));
  String uriPath = relPath.toURI().toString();
  dag.setAttribute(DAGContext.APPLICATION_PATH, uriPath);
  StramLocalCluster cluster = new StramLocalCluster(dag);
  // no need for run(), just need the initial checkpoint
  Assert.assertFalse(cluster.isFinished());
  Assert.assertTrue("app path exists", relPath.exists() && relPath.isDirectory());
  File checkPointDir = new File(relPath, LogicalPlan.SUBDIR_CHECKPOINTS);
  Assert.assertTrue("checkpoint path exists", checkPointDir.exists() && checkPointDir.isDirectory());
}
 

/**
 *
 * @param context
 */
public AppInfo(StramAppContext context)
{
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName();
  this.docLink = context.getApplicationDocLink();
  this.user = context.getUser().toString();
  this.startTime = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startTime, 0);
  this.appPath = context.getApplicationPath();
  this.appMasterTrackingUrl = context.getAppMasterTrackingUrl();
  this.stats = context.getStats();
  this.gatewayAddress = context.getGatewayAddress();
  this.version = VersionInfo.APEX_VERSION.getBuildVersion();
  this.attributes = new TreeMap<>();
  for (Map.Entry<Attribute<Object>, Object> entry : AttributeMap.AttributeInitializer.getAllAttributes(context, DAGContext.class).entrySet()) {
    this.attributes.put(entry.getKey().getSimpleName(), entry.getKey().codec.toString(entry.getValue()));
  }
  this.gatewayConnected = context.isGatewayConnected();
  this.appDataSources = context.getAppDataSources();
  this.metrics = context.getMetrics();
}
 

public static String getGatewayAddress(DAG dag, Configuration conf)
{
  String gatewayAddress = dag.getValue(DAGContext.GATEWAY_CONNECT_ADDRESS);
  if (gatewayAddress == null) {
    gatewayAddress = conf.get(PROP_GATEWAY_ADDRESS);
  }
  return gatewayAddress;
}
 

@Override
public void setup(OperatorContext context)
{
  contextId = context.getValue(DAGContext.APPLICATION_NAME);
  outputFileName = File.separator + contextId +
                   File.separator + "transactions.out.part";
  super.setup(context);
}
 

public static String getGatewayAddress(DAG dag, Configuration conf)
{
  String gatewayAddress = dag.getValue(DAGContext.GATEWAY_CONNECT_ADDRESS);
  if (gatewayAddress == null) {
    gatewayAddress = conf.get(PROP_GATEWAY_ADDRESS);
  }
  return gatewayAddress;
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  TestGeneratorInputOperator input = dag.addOperator("Input", new TestGeneratorInputOperator());
  test = dag.addOperator("Test", new DynamicLoader());

  dag.addStream("S1", input.outport, test.input);
  dag.setAttribute(Context.DAGContext.LIBRARY_JARS, generatedJar);
  dag.setInputPortAttribute(test.input, Context.PortContext.TUPLE_CLASS, pojo);
}
 

private void dagOperatorAttributeHelper(boolean attrOnDag)
{
  String attributeName = null;

  if (attrOnDag) {
    attributeName = DAGContext.CHECKPOINT_WINDOW_COUNT.getSimpleName();
  } else {
    attributeName = OperatorContext.class.getCanonicalName() + LogicalPlanConfiguration.KEY_SEPARATOR + DAGContext.CHECKPOINT_WINDOW_COUNT.getSimpleName();
  }

  Properties props = new Properties();
  String propName = StreamingApplication.APEX_PREFIX + StramElement.ATTR.getValue() + LogicalPlanConfiguration.KEY_SEPARATOR + attributeName;
  props.put(propName, "5");

  SimpleTestApplicationWithName app = new SimpleTestApplicationWithName();

  LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
  dagBuilder.addFromProperties(props, null);

  String appPath = app.getClass().getName().replace(".", "/") + ".class";

  LogicalPlan dag = new LogicalPlan();
  dagBuilder.prepareDAG(dag, app, appPath);

  OperatorMeta om1 = dag.getOperatorMeta("operator1");

  if (attrOnDag) {
    Assert.assertNotEquals((Integer)5, om1.getValue(OperatorContext.CHECKPOINT_WINDOW_COUNT));
  } else {
    Assert.assertEquals((Integer)5, om1.getValue(OperatorContext.CHECKPOINT_WINDOW_COUNT));
  }
}
 

/**
 * Saves the yarn application id which can be used by create application
 * specific table/region in KeyValue sore.
 */
@Override
public void setApplicationAttributes(AttributeMap map)
{
  this.applicationId = map.get(DAGContext.APPLICATION_ID);
  getStore().setTableName(applicationId);
}
 

@Override
public void setup(OperatorContext ctxt)
{
  operatorId = ctxt.getId();
  appId = ctxt.getValue(DAGContext.APPLICATION_ID);
  String v = get(getEndWindowKey());
  if (v != null) {
    committedWindowId = Long.valueOf(v);
  }
}
 

@Test
public void testAttributesCodec()
{
  Assert.assertNotSame(null, new Long[] {com.datatorrent.api.Context.DAGContext.serialVersionUID, OperatorContext.serialVersionUID, PortContext.serialVersionUID});
  @SuppressWarnings("unchecked")
  Set<Class<? extends Context>> contextClasses = Sets.newHashSet(com.datatorrent.api.Context.DAGContext.class, OperatorContext.class, PortContext.class);
  for (Class<?> c : contextClasses) {
    for (Attribute<Object> attr : AttributeInitializer.getAttributes(c)) {
      Assert.assertNotNull(attr.name + " codec", attr.codec);
    }
  }
}
 

@Test
public void testAppAnnotationAlias()
{
  StreamingApplication app = new AnnotatedApplication();
  Configuration conf = new Configuration(false);
  conf.addResource(StramClientUtils.DT_SITE_XML_FILE);

  LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);

  LogicalPlan dag = new LogicalPlan();
  String appPath = app.getClass().getName().replace(".", "/") + ".class";
  builder.prepareDAG(dag, app, appPath);

  Assert.assertEquals("Application name", "AnnotatedAlias", dag.getAttributes().get(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME));
}
 

public static OperatorContext createOperatorContext(int streamingWindowMillis, int appWindowCount)
{
  DefaultAttributeMap attributeMap = new DefaultAttributeMap();
  attributeMap.put(DAGContext.STREAMING_WINDOW_SIZE_MILLIS, streamingWindowMillis);
  attributeMap.put(OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCount);

  return mockOperatorContext(1, attributeMap);
}
 

@Test
public void testBlocksPath()
{
  Assert.assertEquals("Blocks path not initialized in application context",
      context.getValue(DAGContext.APPLICATION_PATH) + Path.SEPARATOR + BlockWriter.DEFAULT_BLOCKS_DIR + Path.SEPARATOR,
      testFM.blocksDir);
}
 

@Test
public void testDeprecation()
{
  String value = "bar";
  String oldKey = StreamingApplication.DT_PREFIX + Context.DAGContext.APPLICATION_NAME.getName();
  String newKey = LogicalPlanConfiguration.KEY_APPLICATION_NAME;
  Configuration config = new Configuration(false);
  config.set(oldKey, value);
  Assert.assertEquals(value, config.get(newKey));
}
 

private void init()
{
  Map<Class<?>, Class<? extends StringCodec<?>>> codecs = logicalPlan.getAttributes().get(DAGContext.STRING_CODECS);
  if (codecs != null) {
    StringCodecs.loadConverters(codecs);
  }
}
 

/**
 * Populate the logical plan from the streaming application definition and configuration.
 * Configuration is resolved based on application alias, if any.
 * @param app The {@link StreamingApplication} to be run.
 * @param dag This will hold the {@link LogicalPlan} representation of the given {@link StreamingApplication}.
 * @param name The path of the application class in the jar.
 */
public void prepareDAG(LogicalPlan dag, StreamingApplication app, String name)
{
  prepareDAGAttributes(dag);

  pluginManager.setup(dag);
  if (app != null) {
    pluginManager.dispatch(PRE_POPULATE_DAG.event);
    app.populateDAG(dag, conf);
    pluginManager.dispatch(POST_POPULATE_DAG.event);
  }
  pluginManager.dispatch(PRE_CONFIGURE_DAG.event);
  String appAlias = getAppAlias(name);
  String appName = appAlias == null ? name : appAlias;
  List<AppConf> appConfs = stramConf.getMatchingChildConf(appName, StramElement.APPLICATION);
  setApplicationConfiguration(dag, appConfs, app);
  if (dag.getAttributes().get(Context.DAGContext.APPLICATION_NAME) == null) {
    dag.setAttribute(Context.DAGContext.APPLICATION_NAME, appName);
  }

  // Expand the modules within the dag recursively
  setModuleProperties(dag, appName);
  flattenDAG(dag, conf);

  // inject external operator configuration
  setOperatorConfiguration(dag, appConfs, appName);
  setStreamConfiguration(dag, appConfs, appName);
  pluginManager.dispatch(POST_CONFIGURE_DAG.event);
  pluginManager.teardown();
}
 

@SuppressWarnings({"rawtypes", "unchecked"})
private void init()
{
  //clear content type
  httpResponse.setContentType(null);
  if (!initialized) {
    Map<Class<?>, Class<? extends StringCodec<?>>> codecs = dagManager.getApplicationAttributes().get(DAGContext.STRING_CODECS);
    StringCodecs.loadConverters(codecs);
    if (codecs != null) {
      SimpleModule sm = new SimpleModule("DTSerializationModule", new Version(1, 0, 0, null));
      for (Map.Entry<Class<?>, Class<? extends StringCodec<?>>> entry : codecs.entrySet()) {
        try {
          final StringCodec<Object> codec = (StringCodec<Object>)entry.getValue().newInstance();
          sm.addSerializer(new SerializerBase(entry.getKey())
          {
            @Override
            public void serialize(Object value, JsonGenerator jgen, SerializerProvider provider) throws IOException, JsonProcessingException
            {
              jgen.writeString(codec.toString(value));
            }

          });
        } catch (Exception ex) {
          LOG.error("Caught exception when instantiating codec for class {}", entry.getKey().getName(), ex);
        }
      }

      objectMapper.registerModule(sm);
    }
    initialized = true;
  }
}
 

private JSONObject getMetricsSchemaData(LogicalPlan.OperatorMeta operatorMeta, Map<String, Object> aggregates)
{
  JSONObject result = new JSONObject();
  try {
    result.put("type", METRICS_SCHEMA);
    result.put("version", METRICS_SCHEMA_VERSION);
    result.put("appUser", appContext.getUser());
    result.put("appName", dnmgr.getApplicationAttributes().get(DAGContext.APPLICATION_NAME));
    result.put("logicalOperatorName", operatorMeta.getName());

    MetricAggregatorMeta metricAggregatorMeta = operatorMeta.getMetricAggregatorMeta();
    JSONArray valueSchemas = new JSONArray();
    for (Map.Entry<String, Object> entry : aggregates.entrySet()) {
      String metricName = entry.getKey();
      Object metricValue = entry.getValue();
      JSONObject valueSchema = new JSONObject();
      valueSchema.put("name", metricName);
      Class<?> type = ClassUtils.wrapperToPrimitive(metricValue.getClass());
      valueSchema.put("type", type == null ? metricValue.getClass().getCanonicalName() : type);
      String[] dimensionAggregators = metricAggregatorMeta.getDimensionAggregatorsFor(metricName);
      if (dimensionAggregators != null) {
        valueSchema.put("dimensionAggregators", Arrays.asList(dimensionAggregators));
      }
      valueSchemas.put(valueSchema);
    }
    result.put("values", valueSchemas);
    String[] timeBuckets = metricAggregatorMeta.getTimeBuckets();
    if (timeBuckets != null) {
      result.put("timeBuckets", Arrays.asList(timeBuckets));
    }

  } catch (JSONException ex) {
    throw new RuntimeException(ex);
  }
  return result;
}
 

public void init()
{
  metricsTransport = dnmgr.getLogicalPlan().getValue(DAGContext.METRICS_TRANSPORT);
  if (metricsTransport instanceof AutoMetricBuiltInTransport) {
    AutoMetricBuiltInTransport transport = (AutoMetricBuiltInTransport)metricsTransport;
    metricsTransport = new PubSubWebSocketMetricTransport(dnmgr.getWsClient(), transport.getTopic(), transport.getSchemaResendInterval());
  }
  LOG.info("Metrics Transport set up for {}", metricsTransport);
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  Generator generator = dag.addOperator("Generator", new Generator());
  HDHTTestOperator store = dag.addOperator("Store", new HDHTTestOperator());
  dag.setAttribute(DAGContext.CHECKPOINT_WINDOW_COUNT, 2);
  //store.setFileStore(new MockFileAccess());
  dag.addStream("Generator2Store", generator.output, store.input);
}
 

@Test
public void testLoadFromJson() throws Exception
{
  String resourcePath = "/testTopology.json";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    fail("Could not load " + resourcePath);
  }
  StringWriter writer = new StringWriter();

  IOUtils.copy(is, writer);
  JSONObject json = new JSONObject(writer.toString());

  Configuration conf = new Configuration(false);
  conf.set(StreamingApplication.APEX_PREFIX + "operator.operator3.prop.myStringProperty", "o3StringFromConf");

  LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
  dag.validate();

  assertEquals("DAG attribute CONTAINER_JVM_OPTIONS ", dag.getAttributes().get(DAGContext.CONTAINER_JVM_OPTIONS), "-Xmx16m");
  Map<Class<?>, Class<? extends StringCodec<?>>> stringCodecsMap = Maps.newHashMap();
  stringCodecsMap.put(Integer.class, Integer2String.class);
  assertEquals("DAG attribute STRING_CODECS ", stringCodecsMap, dag.getAttributes().get(DAGContext.STRING_CODECS));
  assertEquals("DAG attribute CONTAINER_OPTS_CONFIGURATOR ", BasicContainerOptConfigurator.class, dag.getAttributes().get(DAGContext.CONTAINER_OPTS_CONFIGURATOR).getClass());

  assertEquals("number of operator confs", 5, dag.getAllOperators().size());
  assertEquals("number of root operators", 1, dag.getRootOperators().size());

  StreamMeta s1 = dag.getStream("n1n2");
  assertNotNull(s1);
  assertTrue("n1n2 inline", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());

  OperatorMeta input = dag.getOperatorMeta("inputOperator");
  TestStatsListener tsl = new TestStatsListener();
  tsl.setIntProp(222);
  List<StatsListener> sll = Lists.<StatsListener>newArrayList(tsl);
  assertEquals("inputOperator STATS_LISTENERS attribute ", sll, input.getAttributes().get(OperatorContext.STATS_LISTENERS));
  for (OutputPortMeta opm : input.getOutputStreams().keySet()) {
    assertTrue("output port of input Operator attribute is JsonStreamCodec ", opm.getAttributes().get(PortContext.STREAM_CODEC) instanceof JsonStreamCodec<?>);
  }

  OperatorMeta operator3 = dag.getOperatorMeta("operator3");
  assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());

  GenericTestOperator doperator3 = (GenericTestOperator)operator3.getOperator();
  assertEquals("myStringProperty " + doperator3, "o3StringFromConf", doperator3.getMyStringProperty());
  assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);

  OperatorMeta operator4 = dag.getOperatorMeta("operator4");
  GenericTestOperator doperator4 = (GenericTestOperator)operator4.getOperator();
  assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
  assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
  assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);

  StreamMeta input1 = dag.getStream("inputStream");
  assertNotNull(input1);
  OperatorMeta inputOperator = dag.getOperatorMeta("inputOperator");
  Assert.assertEquals("input1 source", inputOperator, input1.getSource().getOperatorMeta());
  Set<OperatorMeta> targetNodes = Sets.newHashSet();
  for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
    targetNodes.add(targetPort.getOperatorMeta());
  }
  Assert.assertEquals("operator attribute " + inputOperator, 64, (int)inputOperator.getValue(OperatorContext.MEMORY_MB));
  Assert.assertEquals("port attribute " + inputOperator, 8, (int)input1.getSource().getValue(PortContext.UNIFIER_LIMIT));
  Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
}
 

@Test
public void testAddOperatorWithAffinityRules()
{
  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
  GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);

  dag.addStream("o1.outport1", o1.outport1, o2.inport1);
  dag.addStream("o2.outport1", o2.outport1, o3.inport1);

  TestPlanContext ctx = new TestPlanContext();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
  PhysicalPlan plan = new PhysicalPlan(dag, ctx);
  ctx.deploy.clear();
  ctx.undeploy.clear();

  Assert.assertEquals("containers", 3, plan.getContainers().size());

  AffinityRulesSet ruleSet = new AffinityRulesSet();
  List<AffinityRule> rules = new ArrayList<>();
  ruleSet.setAffinityRules(rules);
  rules.add(new AffinityRule(Type.AFFINITY, Locality.CONTAINER_LOCAL, false, "o1", "added1"));
  rules.add(new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "o3", "added1"));
  dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);

  PlanModifier pm = new PlanModifier(plan);
  GenericTestOperator added1 = new GenericTestOperator();
  pm.addOperator("added1", added1);

  pm.addStream("added1.outport1", added1.outport1, o3.inport2);

  Assert.assertEquals("undeploy " + ctx.undeploy, 0, ctx.undeploy.size());
  Assert.assertEquals("deploy " + ctx.deploy, 0, ctx.deploy.size());

  pm.applyChanges(ctx);

  Assert.assertEquals("containers post change", 4, plan.getContainers().size());

  Assert.assertEquals("undeploy " + ctx.undeploy, 1, ctx.undeploy.size());
  Assert.assertEquals("deploy " + ctx.deploy, 2, ctx.deploy.size());

  // Validate affinity rules are applied
  for (PTContainer c : plan.getContainers()) {
    if (c.getOperators().contains("added1")) {
      Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", 2, c.getOperators().size());
      Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", "o1", c.getOperators().get(0).getOperatorMeta().getName());
      Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", "added1", c.getOperators().get(1).getOperatorMeta().getName());

      Set<PTContainer> antiAffinityList = c.getStrictAntiPrefs();
      Assert.assertEquals("There should be one container in antiaffinity list", 1, antiAffinityList.size());
      List<PTOperator> antiAffinityOperators = antiAffinityList.iterator().next().getOperators();
      Assert.assertEquals("AntiAffinity operators should containn operator O3", antiAffinityOperators.iterator().next().getOperatorMeta().getName(), "o3");
    }
  }
}
 

@Test
public void testAntiAffinityInOperators()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, testMeta.getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("O1", GenericTestOperator.class);
  dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);

  GenericTestOperator o2 = dag.addOperator("O2", GenericTestOperator.class);
  dag.setOperatorAttribute(o2, OperatorContext.MEMORY_MB, 256);

  dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
  AffinityRulesSet ruleSet = new AffinityRulesSet();

  List<AffinityRule> rules = new ArrayList<>();
  ruleSet.setAffinityRules(rules);
  AffinityRule rule1 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O1", "O2");
  rules.add(rule1);
  dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);

  dag.addStream("o1_outport1", o1.outport1, o2.inport1);// .setLocality(Locality.NODE_LOCAL);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));

  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    csr.container.host = host;
    if (csr.container.getOperators().get(0).getName().equals("O1")) {
      Assert.assertEquals("Hosts set to host1 for Operator O1", "host1", host);
    }
    if (csr.container.getOperators().get(0).getName().equals("O2")) {
      Assert.assertEquals("Hosts set to host2 for Operator O2", "host2", host);
    }
  }
}
 

public void populateDimensionsDAG(DAG dag, Configuration conf, DefaultOutputPort<DimensionTuple> upstreamPort) 
{
  final String eventSchema = SchemaUtils.jarResourceFileToString(eventSchemaLocation);
  
  // dimension
  DimensionsComputationFlexibleSingleSchemaPOJO dimensions = dag.addOperator("DimensionsComputation",
      DimensionsComputationFlexibleSingleSchemaPOJO.class);

  // Set operator properties
  // key expression
  {
    Map<String, String> keyToExpression = Maps.newHashMap();
    keyToExpression.put("campaignId", DimensionTuple.CAMPAIGNID);
    keyToExpression.put("time", DimensionTuple.EVENTTIME);
    dimensions.setKeyToExpression(keyToExpression);
  }

  // aggregate expression
  {
    Map<String, String> valueToExpression = Maps.newHashMap();
    valueToExpression.put("clicks", DimensionTuple.CLICKS);
    valueToExpression.put("latency", DimensionTuple.LATENCY);
    
    dimensions.setAggregateToExpression(valueToExpression);
  }

  // event schema
  dimensions.setConfigurationSchemaJSON(eventSchema);
  dimensions.setUnifier(new DimensionsComputationUnifierImpl<InputEvent, Aggregate>());

  dag.setUnifierAttribute(dimensions.output, OperatorContext.MEMORY_MB, 10240);
  
  dag.setInputPortAttribute(dimensions.input, Context.PortContext.PARTITION_PARALLEL, true);
  
  // store
  AppDataSingleSchemaDimensionStoreHDHT store = createStore(dag, conf, eventSchema); 
  store.setCacheWindowDuration(10000 * 5 / STREAMING_WINDOW_SIZE_MILLIS);   //cache for 5 windows
  dag.addStream("GenerateStream", upstreamPort, dimensions.input).setLocality(Locality.CONTAINER_LOCAL);
  
  StoreStreamCodec codec = new StoreStreamCodec();
  dag.setInputPortAttribute(store.input, PortContext.STREAM_CODEC, codec);
  dag.addStream("DimensionalStream", dimensions.output, store.input);

  
  if (includeQuery) {
    createQuery(dag, conf, store);

    // wsOut
    PubSubWebSocketAppDataResult wsOut = createQueryResult(dag, conf, store);

    dag.addStream("QueryResult", store.queryResult, wsOut.input);
  } else {
    DevNull devNull = new DevNull();
    dag.addOperator("devNull", devNull);
    dag.addStream("QueryResult", store.queryResult, devNull.data);
  }
  
  dag.setAttribute(DAGContext.STREAMING_WINDOW_SIZE_MILLIS, STREAMING_WINDOW_SIZE_MILLIS);
}
 

@Override
protected void serviceStart() throws Exception
{
  super.serviceStart();
  if (UserGroupInformation.isSecurityEnabled()) {
    delegationTokenManager.startThreads();
  }

  // write the connect address for containers to DFS
  InetSocketAddress connectAddress = NetUtils.getConnectAddress(this.heartbeatListener.getAddress());
  URI connectUri = RecoverableRpcProxy.toConnectURI(connectAddress);
  FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), getConfig());
  recoveryHandler.writeConnectUri(connectUri.toString());

  // start web service
  try {
    org.mortbay.log.Log.setLog(null);
  } catch (Throwable throwable) {
    // SPOI-2687. As part of Pivotal Certification, we need to catch ClassNotFoundException as Pivotal was using
    // Jetty 7 where as other distros are using Jetty 6.
    // LOG.error("can't set the log to null: ", throwable);
  }

  try {
    Configuration config = SecurityUtils.configureWebAppSecurity(getConfig(), dag.getValue(Context.DAGContext.SSL_CONFIG));
    WebApp webApp = WebApps.$for("stram", StramAppContext.class, appContext, "ws").with(config).start(new StramWebApp(this.dnmgr));
    LOG.info("Started web service at port: " + webApp.port());
    // best effort to produce FQDN for the client to connect with
    // (when SSL is enabled, it may be required to match the certificate)
    connectAddress = NetUtils.getConnectAddress(webApp.getListenerAddress());
    String hostname = connectAddress.getAddress().getCanonicalHostName();
    if (hostname.equals(connectAddress.getAddress().getHostAddress())) {
      // lookup didn't yield a name
      hostname = connectAddress.getHostName();
    }
    appMasterTrackingUrl = hostname + ":" + webApp.port();
    if (ConfigUtils.isSSLEnabled(config)) {
      appMasterTrackingUrl = "https://" + appMasterTrackingUrl;
    }
    LOG.info("Setting tracking URL to: " + appMasterTrackingUrl);
  } catch (Exception e) {
    LOG.error("Webapps failed to start. Ignoring for now:", e);
  }
}