Java Code Examples for com.datatorrent.stram.plan.logical.LogicalPlan#getAllOperators()

The following examples show how to use com.datatorrent.stram.plan.logical.LogicalPlan#getAllOperators() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: LogicalPlanSerializer.java From Bats with Apache License 2.0

4 votes

public static PropertiesConfiguration convertToProperties(LogicalPlan dag)
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  Collection<OperatorMeta> allOperators = dag.getAllOperators();

  for (OperatorMeta operatorMeta : allOperators) {
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
    Operator operator = operatorMeta.getOperator();
    props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
    BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
    @SuppressWarnings("rawtypes")
    Iterator entryIterator = operatorProperties.entryIterator();
    while (entryIterator.hasNext()) {
      try {
        @SuppressWarnings("unchecked")
        Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next();
        if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
          props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
        }
      } catch (Exception ex) {
        LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
      }
    }
  }
  Collection<StreamMeta> allStreams = dag.getAllStreams();

  for (StreamMeta streamMeta : allStreams) {
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
    OutputPortMeta source = streamMeta.getSource();
    Collection<InputPortMeta> sinks = streamMeta.getSinks();
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
    String sinksValue = "";
    for (InputPortMeta sink : sinks) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    if (streamMeta.getLocality() != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
    }
  }

  // TBD: Attributes

  return props;
}

Example 2

Source File: LogicalPlanSerializer.java From attic-apex-core with Apache License 2.0

4 votes

public static PropertiesConfiguration convertToProperties(LogicalPlan dag)
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  Collection<OperatorMeta> allOperators = dag.getAllOperators();

  for (OperatorMeta operatorMeta : allOperators) {
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
    Operator operator = operatorMeta.getOperator();
    props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
    BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
    @SuppressWarnings("rawtypes")
    Iterator entryIterator = operatorProperties.entryIterator();
    while (entryIterator.hasNext()) {
      try {
        @SuppressWarnings("unchecked")
        Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next();
        if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
          props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
        }
      } catch (Exception ex) {
        LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
      }
    }
  }
  Collection<StreamMeta> allStreams = dag.getAllStreams();

  for (StreamMeta streamMeta : allStreams) {
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
    OutputPortMeta source = streamMeta.getSource();
    Collection<InputPortMeta> sinks = streamMeta.getSinks();
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
    String sinksValue = "";
    for (InputPortMeta sink : sinks) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    if (streamMeta.getLocality() != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
    }
  }

  // TBD: Attributes

  return props;
}

Example 3

Source File: TestModuleExpansion.java From attic-apex-core with Apache License 2.0

4 votes

private void validateTopLevelOperators(LogicalPlan dag)
{
  List<String> operatorNames = new ArrayList<>();
  for (LogicalPlan.OperatorMeta operatorMeta : dag.getAllOperators()) {
    operatorNames.add(operatorMeta.getName());
  }
  Assert.assertTrue(operatorNames.contains("O1"));
  Assert.assertTrue(operatorNames.contains("O2"));
  Assert.assertTrue(operatorNames.contains(componentName("Ma", "M1", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Ma", "M2", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Ma", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Mb", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Mb", "M1", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Mb", "O2")));
  Assert.assertTrue(operatorNames.contains(componentName("Mc", "M1", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Mc", "M2", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Mc", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Md", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Md", "M1", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Md", "O2")));

  validateOperatorPropertyValue(dag, "O1", 1);
  validateOperatorPropertyValue(dag, "O2", 2);
  validateOperatorPropertyValue(dag, componentName("Ma", "M1", "O1"), 11);
  validateOperatorPropertyValue(dag, componentName("Ma", "M2", "O1"), 12);
  validateOperatorPropertyValue(dag, componentName("Ma", "O1"), 13);
  validateOperatorPropertyValue(dag, componentName("Mb", "O1"), 21);
  validateOperatorPropertyValue(dag, componentName("Mb", "M1", "O1"), 22);
  validateOperatorPropertyValue(dag, componentName("Mb", "O2"), 23);
  validateOperatorPropertyValue(dag, componentName("Mc", "M1", "O1"), 31);
  validateOperatorPropertyValue(dag, componentName("Mc", "M2", "O1"), 32);
  validateOperatorPropertyValue(dag, componentName("Mc", "O1"), 33);
  validateOperatorPropertyValue(dag, componentName("Md", "O1"), 41);
  validateOperatorPropertyValue(dag, componentName("Md", "M1", "O1"), 42);
  validateOperatorPropertyValue(dag, componentName("Md", "O2"), 43);

  validateOperatorParent(dag, "O1", null);
  validateOperatorParent(dag, "O2", null);
  validateOperatorParent(dag, componentName("Ma", "M1", "O1"), componentName("Ma", "M1"));
  validateOperatorParent(dag, componentName("Ma", "M2", "O1"), componentName("Ma", "M2"));
  validateOperatorParent(dag, componentName("Ma", "O1"), "Ma");
  validateOperatorParent(dag, componentName("Mb", "O1"), "Mb");
  validateOperatorParent(dag, componentName("Mb", "M1", "O1"), componentName("Mb", "M1"));
  validateOperatorParent(dag, componentName("Mb", "O2"), "Mb");
  validateOperatorParent(dag, componentName("Mc", "M1", "O1"), componentName("Mc", "M1"));
  validateOperatorParent(dag, componentName("Mc", "M2", "O1"), componentName("Mc", "M2"));
  validateOperatorParent(dag, componentName("Mc", "O1"), "Mc");
  validateOperatorParent(dag, componentName("Md", "O1"), "Md");
  validateOperatorParent(dag, componentName("Md", "M1", "O1"), componentName("Md", "M1"));
  validateOperatorParent(dag, componentName("Md", "O2"), "Md");

  validateOperatorAttribute(dag, componentName("Ma", "M1", "O1"), 1024);
  validateOperatorAttribute(dag, componentName("Ma", "M2", "O1"), 2048);
  validateOperatorAttribute(dag, componentName("Mb", "M1", "O1"), 4096);
  validateOperatorAttribute(dag, componentName("Mc", "M1", "O1"), 1024);
  validateOperatorAttribute(dag, componentName("Mc", "M2", "O1"), 2048);

  validatePortAttribute(dag, componentName("Ma", "M1", "O1"), 1);
  validatePortAttribute(dag, componentName("Ma", "M2", "O1"), 2);
  validatePortAttribute(dag, componentName("Mb", "M1", "O1"), 3);
  validatePortAttribute(dag, componentName("Mc", "M1", "O1"), 1);
  validatePortAttribute(dag, componentName("Mc", "M2", "O1"), 2);
}

Example 4

Source File: ApexStreamImplTest.java From attic-apex-malhar with Apache License 2.0

4 votes

@Test
public void testAddOperator()
{
  LogicalPlan dag = new LogicalPlan();
  TestOperator<String, Integer> firstOperator = new TestOperator<>();
  TestOperator<Integer, Date> secondOperator = new TestOperator<>();
  new ApexStreamImpl<String>().addOperator(firstOperator, null, firstOperator.output, name("first"))
      .endWith(secondOperator, secondOperator.input, name("second"))
      .with(DAG.Locality.THREAD_LOCAL)
      .with(Context.OperatorContext.AUTO_RECORD, true)
      .with("prop", "TestProp").populateDag(dag);
  Assert.assertTrue(dag.getAllOperators().size() == 2);
  Set<String> opNames = new HashSet<>();
  opNames.add("first");
  opNames.add("second");
  for (LogicalPlan.OperatorMeta operatorMeta : dag.getAllOperators()) {
    Assert.assertTrue(operatorMeta.getOperator() instanceof TestOperator);
    Assert.assertTrue(opNames.contains(operatorMeta.getName()));
    if (operatorMeta.getName().equals("second")) {
      // Assert the Context.OperatorContext.AUTO_RECORD attribute has been set to true for second operator
      Assert.assertTrue(operatorMeta.getAttributes().get(Context.OperatorContext.AUTO_RECORD));
      // Assert the prop has been set to TestProp for second operator
      Assert.assertEquals("TestProp", ((TestOperator)operatorMeta.getOperator()).prop);
    } else {
      // Assert the Context.OperatorContext.AUTO_RECORD attribute keeps as default false for first operator
      Assert.assertNull(operatorMeta.getAttributes().get(Context.OperatorContext.AUTO_RECORD));
      // Assert the prop has not been set for first operator
      Assert.assertNull(((TestOperator)operatorMeta.getOperator()).prop);
    }
  }

  Collection<LogicalPlan.StreamMeta> streams = dag.getAllStreams();
  // Assert there is only one stream
  Assert.assertTrue(streams.size() == 1);
  for (LogicalPlan.StreamMeta stream : streams) {

    // Assert the stream is from first operator to second operator
    Assert.assertEquals("first", stream.getSource().getOperatorMeta().getName());
    Collection<InputPortMeta> portMetaCollection = stream.getSinks();
    Assert.assertTrue(1 == portMetaCollection.size());
    for (InputPortMeta inputPortMeta : portMetaCollection) {
      Assert.assertEquals("second", inputPortMeta.getOperatorMeta().getName());
    }

    // Assert the stream is thread local
    Assert.assertTrue(stream.getLocality() == DAG.Locality.THREAD_LOCAL);
  }

}