Java Code Examples for com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta#getOperator()

public static PTOperator createSlidingUnifier(StreamMeta streamMeta, PhysicalPlan plan, int
    operatorApplicationWindowCount, int slidingWindowCount)
{
  int gcd = IntMath.gcd(operatorApplicationWindowCount, slidingWindowCount);
  OperatorMeta um = streamMeta.getSource()
      .getSlidingUnifier(operatorApplicationWindowCount / gcd, gcd, slidingWindowCount / gcd);
  PTOperator pu = plan.newOperator(um, um.getName());

  Operator unifier = um.getOperator();
  PortMappingDescriptor mergeDesc = new PortMappingDescriptor();
  Operators.describe(unifier, mergeDesc);
  if (mergeDesc.outputPorts.size() != 1) {
    throw new AssertionError("Unifier must have a single output port, instead found : " + mergeDesc.outputPorts);
  }
  pu.unifiedOperatorMeta = streamMeta.getSource().getOperatorMeta();
  pu.outputs.add(new PTOutput(mergeDesc.outputPorts.keySet().iterator().next(), streamMeta, pu));
  plan.newOpers.put(pu, unifier);
  return pu;
}
 

public static PTOperator createUnifier(StreamMeta streamMeta, PhysicalPlan plan)
{
  OperatorMeta um = streamMeta.getSource().getUnifierMeta();
  PTOperator pu = plan.newOperator(um, um.getName());

  Operator unifier = um.getOperator();
  PortMappingDescriptor mergeDesc = new PortMappingDescriptor();
  Operators.describe(unifier, mergeDesc);
  if (mergeDesc.outputPorts.size() != 1) {
    throw new AssertionError("Unifier must have a single output port, instead found : " + mergeDesc.outputPorts);
  }

  pu.unifiedOperatorMeta = streamMeta.getSource().getOperatorMeta();
  pu.outputs.add(new PTOutput(mergeDesc.outputPorts.keySet().iterator().next(), streamMeta, pu));
  plan.newOpers.put(pu, unifier);
  return pu;
}
 

public static PTOperator createSlidingUnifier(StreamMeta streamMeta, PhysicalPlan plan, int
    operatorApplicationWindowCount, int slidingWindowCount)
{
  int gcd = IntMath.gcd(operatorApplicationWindowCount, slidingWindowCount);
  OperatorMeta um = streamMeta.getSource()
      .getSlidingUnifier(operatorApplicationWindowCount / gcd, gcd, slidingWindowCount / gcd);
  PTOperator pu = plan.newOperator(um, um.getName());

  Operator unifier = um.getOperator();
  PortMappingDescriptor mergeDesc = new PortMappingDescriptor();
  Operators.describe(unifier, mergeDesc);
  if (mergeDesc.outputPorts.size() != 1) {
    throw new AssertionError("Unifier must have a single output port, instead found : " + mergeDesc.outputPorts);
  }
  pu.unifiedOperatorMeta = streamMeta.getSource().getOperatorMeta();
  pu.outputs.add(new PTOutput(mergeDesc.outputPorts.keySet().iterator().next(), streamMeta, pu));
  plan.newOpers.put(pu, unifier);
  return pu;
}
 

public static PTOperator createUnifier(StreamMeta streamMeta, PhysicalPlan plan)
{
  OperatorMeta um = streamMeta.getSource().getUnifierMeta();
  PTOperator pu = plan.newOperator(um, um.getName());

  Operator unifier = um.getOperator();
  PortMappingDescriptor mergeDesc = new PortMappingDescriptor();
  Operators.describe(unifier, mergeDesc);
  if (mergeDesc.outputPorts.size() != 1) {
    throw new AssertionError("Unifier must have a single output port, instead found : " + mergeDesc.outputPorts);
  }

  pu.unifiedOperatorMeta = streamMeta.getSource().getOperatorMeta();
  pu.outputs.add(new PTOutput(mergeDesc.outputPorts.keySet().iterator().next(), streamMeta, pu));
  plan.newOpers.put(pu, unifier);
  return pu;
}
 

@Test
public void testLoadFromPropertiesFileWithLegacyPrefix() throws IOException
{
  Properties props = new Properties();
  String resourcePath = "/testTopologyLegacyPrefix.properties";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    fail("Could not load " + resourcePath);
  }
  props.load(is);
  LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false)).addFromProperties(props, null);

  LogicalPlan dag = new LogicalPlan();
  pb.populateDAG(dag);
  dag.validate();

  assertEquals("number of operators", 2, dag.getAllOperators().size());
  assertEquals("number of root operators", 1, dag.getRootOperators().size());

  StreamMeta s1 = dag.getStream("s1");
  assertNotNull(s1);
  assertTrue("s1 locality", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());

  OperatorMeta o2m = dag.getOperatorMeta("o2");
  assertEquals(GenericTestOperator.class, o2m.getOperator().getClass());
  GenericTestOperator o2 = (GenericTestOperator)o2m.getOperator();
  assertEquals("myStringProperty " + o2, "myStringPropertyValue", o2.getMyStringProperty());
}
 

public static PropertiesConfiguration convertToProperties(LogicalPlan dag)
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  Collection<OperatorMeta> allOperators = dag.getAllOperators();

  for (OperatorMeta operatorMeta : allOperators) {
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
    Operator operator = operatorMeta.getOperator();
    props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
    BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
    @SuppressWarnings("rawtypes")
    Iterator entryIterator = operatorProperties.entryIterator();
    while (entryIterator.hasNext()) {
      try {
        @SuppressWarnings("unchecked")
        Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next();
        if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
          props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
        }
      } catch (Exception ex) {
        LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
      }
    }
  }
  Collection<StreamMeta> allStreams = dag.getAllStreams();

  for (StreamMeta streamMeta : allStreams) {
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
    OutputPortMeta source = streamMeta.getSource();
    Collection<InputPortMeta> sinks = streamMeta.getSinks();
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
    String sinksValue = "";
    for (InputPortMeta sink : sinks) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    if (streamMeta.getLocality() != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
    }
  }

  // TBD: Attributes

  return props;
}
 

public static PropertiesConfiguration convertToProperties(LogicalPlan dag)
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  Collection<OperatorMeta> allOperators = dag.getAllOperators();

  for (OperatorMeta operatorMeta : allOperators) {
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
    Operator operator = operatorMeta.getOperator();
    props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
    BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
    @SuppressWarnings("rawtypes")
    Iterator entryIterator = operatorProperties.entryIterator();
    while (entryIterator.hasNext()) {
      try {
        @SuppressWarnings("unchecked")
        Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next();
        if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
          props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
        }
      } catch (Exception ex) {
        LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
      }
    }
  }
  Collection<StreamMeta> allStreams = dag.getAllStreams();

  for (StreamMeta streamMeta : allStreams) {
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
    OutputPortMeta source = streamMeta.getSource();
    Collection<InputPortMeta> sinks = streamMeta.getSinks();
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
    String sinksValue = "";
    for (InputPortMeta sink : sinks) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    if (streamMeta.getLocality() != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
    }
  }

  // TBD: Attributes

  return props;
}
 

@Test
  public void testLogicalPlanConfiguration() throws Exception
  {
    TestOperator<String, Map<String, Number>> bean = new InputTestOperator<>();
    bean.map.put("key1", new Structured());
    bean.stringArray = new String[]{"one", "two", "three"};
    bean.stringList = Lists.newArrayList("four", "five");
    bean.props = new Properties();
    bean.props.setProperty("key1", "value1");
    bean.structuredArray = new Structured[]{new Structured()};
    bean.genericArray = new String[]{"s1"};
    bean.structuredArray[0].name = "s1";
    bean.color = Color.BLUE;
    bean.booleanProp = true;
    bean.realName = "abc";

    ObjectMapper mapper = ObjectMapperFactory.getOperatorValueSerializer();
    String s = mapper.writeValueAsString(bean);
//    LOG.debug(new JSONObject(s).toString(2));
    //
    Assert.assertTrue("Shouldn't contain field 'realName' !", !s.contains("realName"));
    Assert.assertTrue("Should contain property 'alias' !", s.contains("alias"));
    Assert.assertTrue("Shouldn't contain property 'getterOnly' !", !s.contains("getterOnly"));
    JSONObject jsonObj = new JSONObject(s);

    // create the json dag representation
    JSONObject jsonPlan = new JSONObject();
    jsonPlan.put("streams", new JSONArray());
    JSONObject jsonOper = new JSONObject();
    jsonOper.put("name", "Test Operator");
    jsonOper.put("class", InputTestOperator.class.getName());
    jsonOper.put("properties", jsonObj);
    jsonPlan.put("operators", new JSONArray(Lists.newArrayList(jsonOper)));

    Configuration conf = new Configuration(false);
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
    // create logical plan from the json
    LogicalPlan lp = lpc.createFromJson(jsonPlan, "jsontest");
    OperatorMeta om = lp.getOperatorMeta("Test Operator");
    Assert.assertTrue(om.getOperator() instanceof InputTestOperator);
    @SuppressWarnings("rawtypes")
    TestOperator beanBack = (InputTestOperator)om.getOperator();

    // The operator deserialized back from json should be same as original operator
    Assert.assertEquals(bean.map, beanBack.map);
    Assert.assertArrayEquals(bean.stringArray, beanBack.stringArray);
    Assert.assertEquals(bean.stringList, beanBack.stringList);
    Assert.assertEquals(bean.props, beanBack.props);
    Assert.assertArrayEquals(bean.structuredArray, beanBack.structuredArray);
    Assert.assertArrayEquals(bean.genericArray, beanBack.genericArray);
    Assert.assertEquals(bean.color, beanBack.color);
    Assert.assertEquals(bean.booleanProp, beanBack.booleanProp);
    Assert.assertEquals(bean.realName, beanBack.realName);
    Assert.assertEquals(bean.getterOnly, beanBack.getterOnly);

  }
 

/**
 * Test read from configuration file in Hadoop configuration format.
 */
@Test
public void testLoadFromConfigXml()
{
  Configuration conf = new Configuration(false);
  conf.addResource(StramClientUtils.DT_SITE_XML_FILE);

  LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);

  LogicalPlan dag = new LogicalPlan();
  builder.populateDAG(dag);
  dag.validate();

  assertEquals("number of operator confs", 6, dag.getAllOperators().size());

  OperatorMeta operator1 = assertNode(dag, "operator1");
  OperatorMeta operator2 = assertNode(dag, "operator2");
  OperatorMeta operator3 = assertNode(dag, "operator3");
  OperatorMeta operator4 = assertNode(dag, "operator4");

  assertNotNull("operatorConf for root", operator1);
  assertEquals("operatorId set", "operator1", operator1.getName());

  // verify operator instantiation
  assertEquals(operator1.getOperator().getClass(), TestGeneratorInputOperator.class);
  TestGeneratorInputOperator GenericTestNode = (TestGeneratorInputOperator)operator1.getOperator();
  assertEquals("myStringPropertyValue", GenericTestNode.getMyStringProperty());

  // check links
  assertEquals("operator1 inputs", 0, operator1.getInputStreams().size());
  assertEquals("operator1 outputs", 1, operator1.getOutputStreams().size());
  StreamMeta n1n2 = operator2.getInputStreams().get(operator2.getMeta(((GenericTestOperator)operator2.getOperator()).inport1));
  assertNotNull("n1n2", n1n2);

  // output/input stream object same
  assertEquals("rootNode out is operator2 in", n1n2, operator1.getOutputStreams().get(operator1.getMeta(((TestGeneratorInputOperator)operator1.getOperator()).outport)));
  assertEquals("n1n2 source", operator1, n1n2.getSource().getOperatorMeta());
  Assert.assertEquals("n1n2 targets", 1, n1n2.getSinks().size());
  Assert.assertEquals("n1n2 target", operator2, n1n2.getSinks().iterator().next().getOperatorMeta());

  assertEquals("stream name", "n1n2", n1n2.getName());
  Assert.assertEquals("n1n2 not inline (default)", null, n1n2.getLocality());

  // operator 2 streams to operator 3 and operator 4
  assertEquals("operator 2 number of outputs", 1, operator2.getOutputStreams().size());
  StreamMeta fromNode2 = operator2.getOutputStreams().values().iterator().next();

  Set<OperatorMeta> targetNodes = Sets.newHashSet();
  for (LogicalPlan.InputPortMeta ip : fromNode2.getSinks()) {
    targetNodes.add(ip.getOperatorMeta());
  }
  Assert.assertEquals("outputs " + fromNode2, Sets.newHashSet(operator3, operator4), targetNodes);

  OperatorMeta operator6 = assertNode(dag, "operator6");

  List<OperatorMeta> rootNodes = dag.getRootOperators();
  assertEquals("number root operators", 2, rootNodes.size());
  assertTrue("root operator2", rootNodes.contains(operator1));
  assertTrue("root operator6", rootNodes.contains(operator6));

  for (OperatorMeta n : rootNodes) {
    printTopology(n, dag, 0);
  }

}
 

@Test
public void testLoadFromPropertiesFile() throws IOException
{
  Properties props = new Properties();
  String resourcePath = "/testTopology.properties";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    fail("Could not load " + resourcePath);
  }
  props.load(is);
  LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false)).addFromProperties(props, null);

  LogicalPlan dag = new LogicalPlan();
  pb.populateDAG(dag);
  dag.validate();

  assertEquals("number of operator confs", 5, dag.getAllOperators().size());
  assertEquals("number of root operators", 1, dag.getRootOperators().size());

  StreamMeta s1 = dag.getStream("n1n2");
  assertNotNull(s1);
  assertTrue("n1n2 locality", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());

  OperatorMeta operator3 = dag.getOperatorMeta("operator3");
  assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());

  GenericTestOperator doperator3 = (GenericTestOperator)operator3.getOperator();
  assertEquals("myStringProperty " + doperator3, "myStringPropertyValueFromTemplate", doperator3.getMyStringProperty());
  assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);

  OperatorMeta operator4 = dag.getOperatorMeta("operator4");
  GenericTestOperator doperator4 = (GenericTestOperator)operator4.getOperator();
  assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
  assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
  assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);

  StreamMeta input1 = dag.getStream("inputStream");
  assertNotNull(input1);
  Assert.assertEquals("input1 source", dag.getOperatorMeta("inputOperator"), input1.getSource().getOperatorMeta());
  Set<OperatorMeta> targetNodes = Sets.newHashSet();
  for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
    targetNodes.add(targetPort.getOperatorMeta());
  }

  Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);

}
 

@Test
public void testLoadFromJson() throws Exception
{
  String resourcePath = "/testTopology.json";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    fail("Could not load " + resourcePath);
  }
  StringWriter writer = new StringWriter();

  IOUtils.copy(is, writer);
  JSONObject json = new JSONObject(writer.toString());

  Configuration conf = new Configuration(false);
  conf.set(StreamingApplication.APEX_PREFIX + "operator.operator3.prop.myStringProperty", "o3StringFromConf");

  LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
  dag.validate();

  assertEquals("DAG attribute CONTAINER_JVM_OPTIONS ", dag.getAttributes().get(DAGContext.CONTAINER_JVM_OPTIONS), "-Xmx16m");
  Map<Class<?>, Class<? extends StringCodec<?>>> stringCodecsMap = Maps.newHashMap();
  stringCodecsMap.put(Integer.class, Integer2String.class);
  assertEquals("DAG attribute STRING_CODECS ", stringCodecsMap, dag.getAttributes().get(DAGContext.STRING_CODECS));
  assertEquals("DAG attribute CONTAINER_OPTS_CONFIGURATOR ", BasicContainerOptConfigurator.class, dag.getAttributes().get(DAGContext.CONTAINER_OPTS_CONFIGURATOR).getClass());

  assertEquals("number of operator confs", 5, dag.getAllOperators().size());
  assertEquals("number of root operators", 1, dag.getRootOperators().size());

  StreamMeta s1 = dag.getStream("n1n2");
  assertNotNull(s1);
  assertTrue("n1n2 inline", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());

  OperatorMeta input = dag.getOperatorMeta("inputOperator");
  TestStatsListener tsl = new TestStatsListener();
  tsl.setIntProp(222);
  List<StatsListener> sll = Lists.<StatsListener>newArrayList(tsl);
  assertEquals("inputOperator STATS_LISTENERS attribute ", sll, input.getAttributes().get(OperatorContext.STATS_LISTENERS));
  for (OutputPortMeta opm : input.getOutputStreams().keySet()) {
    assertTrue("output port of input Operator attribute is JsonStreamCodec ", opm.getAttributes().get(PortContext.STREAM_CODEC) instanceof JsonStreamCodec<?>);
  }

  OperatorMeta operator3 = dag.getOperatorMeta("operator3");
  assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());

  GenericTestOperator doperator3 = (GenericTestOperator)operator3.getOperator();
  assertEquals("myStringProperty " + doperator3, "o3StringFromConf", doperator3.getMyStringProperty());
  assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);

  OperatorMeta operator4 = dag.getOperatorMeta("operator4");
  GenericTestOperator doperator4 = (GenericTestOperator)operator4.getOperator();
  assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
  assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
  assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);

  StreamMeta input1 = dag.getStream("inputStream");
  assertNotNull(input1);
  OperatorMeta inputOperator = dag.getOperatorMeta("inputOperator");
  Assert.assertEquals("input1 source", inputOperator, input1.getSource().getOperatorMeta());
  Set<OperatorMeta> targetNodes = Sets.newHashSet();
  for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
    targetNodes.add(targetPort.getOperatorMeta());
  }
  Assert.assertEquals("operator attribute " + inputOperator, 64, (int)inputOperator.getValue(OperatorContext.MEMORY_MB));
  Assert.assertEquals("port attribute " + inputOperator, 8, (int)input1.getSource().getValue(PortContext.UNIFIER_LIMIT));
  Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
}