com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta Java Examples

private PTOperator addPTOperator(PMapping nodeDecl, Partition<? extends Operator> partition, Checkpoint checkpoint)
{
  PTOperator oper = newOperator(nodeDecl.logicalOperator, nodeDecl.logicalOperator.getName());
  oper.recoveryCheckpoint = checkpoint;

  // output port objects
  for (Map.Entry<LogicalPlan.OutputPortMeta, StreamMeta> outputEntry : nodeDecl.logicalOperator.getOutputStreams().entrySet()) {
    setupOutput(nodeDecl, oper, outputEntry);
  }

  String host = null;
  if (partition != null) {
    oper.setPartitionKeys(partition.getPartitionKeys());
    host = partition.getAttributes().get(OperatorContext.LOCALITY_HOST);
  }
  if (host == null) {
    host = nodeDecl.logicalOperator.getValue(OperatorContext.LOCALITY_HOST);
  }

  nodeDecl.addPartition(oper);
  this.newOpers.put(oper, partition != null ? partition.getPartitionedInstance() : nodeDecl.logicalOperator.getOperator());

  //
  // update locality
  //
  setLocalityGrouping(nodeDecl, oper, inlinePrefs, Locality.CONTAINER_LOCAL, host);
  setLocalityGrouping(nodeDecl, oper, localityPrefs, Locality.NODE_LOCAL, host);

  return oper;
}
 

private PTOperator addPTOperator(PMapping nodeDecl, Partition<? extends Operator> partition, Checkpoint checkpoint)
{
  PTOperator oper = newOperator(nodeDecl.logicalOperator, nodeDecl.logicalOperator.getName());
  oper.recoveryCheckpoint = checkpoint;

  // output port objects
  for (Map.Entry<LogicalPlan.OutputPortMeta, StreamMeta> outputEntry : nodeDecl.logicalOperator.getOutputStreams().entrySet()) {
    setupOutput(nodeDecl, oper, outputEntry);
  }

  String host = null;
  if (partition != null) {
    oper.setPartitionKeys(partition.getPartitionKeys());
    host = partition.getAttributes().get(OperatorContext.LOCALITY_HOST);
  }
  if (host == null) {
    host = nodeDecl.logicalOperator.getValue(OperatorContext.LOCALITY_HOST);
  }

  nodeDecl.addPartition(oper);
  this.newOpers.put(oper, partition != null ? partition.getPartitionedInstance() : nodeDecl.logicalOperator.getOperator());

  //
  // update locality
  //
  setLocalityGrouping(nodeDecl, oper, inlinePrefs, Locality.CONTAINER_LOCAL, host);
  setLocalityGrouping(nodeDecl, oper, localityPrefs, Locality.NODE_LOCAL, host);

  return oper;
}
 

private void simpleNameOutputPortAssertHelper(Attribute<?> attributeObj, LogicalPlan dag, String operatorName, Object queueCapacity, boolean set)
{
  OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);

  for (OutputPortMeta outputPortMeta: operatorMeta.getOutputStreams().keySet()) {
    if (set) {
      Assert.assertEquals(queueCapacity, outputPortMeta.getValue(attributeObj));
    } else {
      Assert.assertNotEquals(queueCapacity, outputPortMeta.getValue(attributeObj));
    }
  }
}
 

public void dfs(OperatorMeta o, List<OperatorMeta> visited) {
  visited.add(o);

  for (Entry<OutputPortMeta, StreamMeta> opm : o.getOutputStreams().entrySet()) {
    // Samoa won't allow one output port to multiple input port kind of streams
    OperatorMeta downStreamOp = opm.getValue().getSinks().get(0).getOperatorWrapper();
    if (visited.contains(downStreamOp)) {
      loopStreams.add(opm.getValue());
    } else {
      List<OperatorMeta> v2 = Lists.newArrayList();
      v2.addAll(visited);
      dfs(downStreamOp, v2);
    }
  }
}
 

public static PropertiesConfiguration convertToProperties(LogicalPlan dag)
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  Collection<OperatorMeta> allOperators = dag.getAllOperators();

  for (OperatorMeta operatorMeta : allOperators) {
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
    Operator operator = operatorMeta.getOperator();
    props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
    BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
    @SuppressWarnings("rawtypes")
    Iterator entryIterator = operatorProperties.entryIterator();
    while (entryIterator.hasNext()) {
      try {
        @SuppressWarnings("unchecked")
        Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next();
        if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
          props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
        }
      } catch (Exception ex) {
        LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
      }
    }
  }
  Collection<StreamMeta> allStreams = dag.getAllStreams();

  for (StreamMeta streamMeta : allStreams) {
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
    OutputPortMeta source = streamMeta.getSource();
    Collection<InputPortMeta> sinks = streamMeta.getSinks();
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
    String sinksValue = "";
    for (InputPortMeta sink : sinks) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    if (streamMeta.getLocality() != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
    }
  }

  // TBD: Attributes

  return props;
}
 

public static PropertiesConfiguration convertToProperties(LogicalPlan dag)
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  Collection<OperatorMeta> allOperators = dag.getAllOperators();

  for (OperatorMeta operatorMeta : allOperators) {
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
    Operator operator = operatorMeta.getOperator();
    props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
    BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
    @SuppressWarnings("rawtypes")
    Iterator entryIterator = operatorProperties.entryIterator();
    while (entryIterator.hasNext()) {
      try {
        @SuppressWarnings("unchecked")
        Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next();
        if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
          props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
        }
      } catch (Exception ex) {
        LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
      }
    }
  }
  Collection<StreamMeta> allStreams = dag.getAllStreams();

  for (StreamMeta streamMeta : allStreams) {
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
    OutputPortMeta source = streamMeta.getSource();
    Collection<InputPortMeta> sinks = streamMeta.getSinks();
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
    String sinksValue = "";
    for (InputPortMeta sink : sinks) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    if (streamMeta.getLocality() != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
    }
  }

  // TBD: Attributes

  return props;
}
 

@Test
public void testLoadFromJson() throws Exception
{
  String resourcePath = "/testTopology.json";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    fail("Could not load " + resourcePath);
  }
  StringWriter writer = new StringWriter();

  IOUtils.copy(is, writer);
  JSONObject json = new JSONObject(writer.toString());

  Configuration conf = new Configuration(false);
  conf.set(StreamingApplication.APEX_PREFIX + "operator.operator3.prop.myStringProperty", "o3StringFromConf");

  LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
  dag.validate();

  assertEquals("DAG attribute CONTAINER_JVM_OPTIONS ", dag.getAttributes().get(DAGContext.CONTAINER_JVM_OPTIONS), "-Xmx16m");
  Map<Class<?>, Class<? extends StringCodec<?>>> stringCodecsMap = Maps.newHashMap();
  stringCodecsMap.put(Integer.class, Integer2String.class);
  assertEquals("DAG attribute STRING_CODECS ", stringCodecsMap, dag.getAttributes().get(DAGContext.STRING_CODECS));
  assertEquals("DAG attribute CONTAINER_OPTS_CONFIGURATOR ", BasicContainerOptConfigurator.class, dag.getAttributes().get(DAGContext.CONTAINER_OPTS_CONFIGURATOR).getClass());

  assertEquals("number of operator confs", 5, dag.getAllOperators().size());
  assertEquals("number of root operators", 1, dag.getRootOperators().size());

  StreamMeta s1 = dag.getStream("n1n2");
  assertNotNull(s1);
  assertTrue("n1n2 inline", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());

  OperatorMeta input = dag.getOperatorMeta("inputOperator");
  TestStatsListener tsl = new TestStatsListener();
  tsl.setIntProp(222);
  List<StatsListener> sll = Lists.<StatsListener>newArrayList(tsl);
  assertEquals("inputOperator STATS_LISTENERS attribute ", sll, input.getAttributes().get(OperatorContext.STATS_LISTENERS));
  for (OutputPortMeta opm : input.getOutputStreams().keySet()) {
    assertTrue("output port of input Operator attribute is JsonStreamCodec ", opm.getAttributes().get(PortContext.STREAM_CODEC) instanceof JsonStreamCodec<?>);
  }

  OperatorMeta operator3 = dag.getOperatorMeta("operator3");
  assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());

  GenericTestOperator doperator3 = (GenericTestOperator)operator3.getOperator();
  assertEquals("myStringProperty " + doperator3, "o3StringFromConf", doperator3.getMyStringProperty());
  assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);

  OperatorMeta operator4 = dag.getOperatorMeta("operator4");
  GenericTestOperator doperator4 = (GenericTestOperator)operator4.getOperator();
  assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
  assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
  assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);

  StreamMeta input1 = dag.getStream("inputStream");
  assertNotNull(input1);
  OperatorMeta inputOperator = dag.getOperatorMeta("inputOperator");
  Assert.assertEquals("input1 source", inputOperator, input1.getSource().getOperatorMeta());
  Set<OperatorMeta> targetNodes = Sets.newHashSet();
  for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
    targetNodes.add(targetPort.getOperatorMeta());
  }
  Assert.assertEquals("operator attribute " + inputOperator, 64, (int)inputOperator.getValue(OperatorContext.MEMORY_MB));
  Assert.assertEquals("port attribute " + inputOperator, 8, (int)input1.getSource().getValue(PortContext.UNIFIER_LIMIT));
  Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
}