com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta Java Examples

public boolean shouldCaptureEvent(T o)
{
  for (Entry<InputPortMeta, Collection<PartitionKeys>> entry : inputPortToPartitionMap.entrySet()) {
    StreamCodec<Object> codec = codecsToMerge.get(entry.getKey());
    Collection<PartitionKeys> partitionKeysList = entry.getValue();

    for (PartitionKeys keys : partitionKeysList) {
      if ( keys.partitions != null && keys.partitions.contains(keys.mask & codec.getPartition(o))) {
        // Then at least one of the partitions is getting this event
        // So send the event to persist operator
        return true;
      }
    }
  }

  return false;
}
 

private void setInput(PTOperator oper, InputPortMeta ipm, PTOperator sourceOper, PartitionKeys pks)
{
  // TODO: see if this can be handled more efficiently
  for (PTInput in : oper.inputs) {
    if (in.source.source == sourceOper && in.logicalStream == streamMeta && ipm.getPortName().equals(in.portName)) {
      return;
    }
  }
  // link to upstream output(s) for this stream
  for (PTOutput upstreamOut : sourceOper.outputs) {
    if (upstreamOut.logicalStream == streamMeta) {
      PTInput input = new PTInput(ipm.getPortName(), streamMeta, oper, pks, upstreamOut, ipm.getValue(LogicalPlan.IS_CONNECTED_TO_DELAY_OPERATOR));
      oper.inputs.add(input);
    }
  }
}
 

public boolean shouldCaptureEvent(T o)
{
  for (Entry<InputPortMeta, Collection<PartitionKeys>> entry : inputPortToPartitionMap.entrySet()) {
    StreamCodec<Object> codec = codecsToMerge.get(entry.getKey());
    Collection<PartitionKeys> partitionKeysList = entry.getValue();

    for (PartitionKeys keys : partitionKeysList) {
      if ( keys.partitions != null && keys.partitions.contains(keys.mask & codec.getPartition(o))) {
        // Then at least one of the partitions is getting this event
        // So send the event to persist operator
        return true;
      }
    }
  }

  return false;
}
 

private void setInput(PTOperator oper, InputPortMeta ipm, PTOperator sourceOper, PartitionKeys pks)
{
  // TODO: see if this can be handled more efficiently
  for (PTInput in : oper.inputs) {
    if (in.source.source == sourceOper && in.logicalStream == streamMeta && ipm.getPortName().equals(in.portName)) {
      return;
    }
  }
  // link to upstream output(s) for this stream
  for (PTOutput upstreamOut : sourceOper.outputs) {
    if (upstreamOut.logicalStream == streamMeta) {
      PTInput input = new PTInput(ipm.getPortName(), streamMeta, oper, pks, upstreamOut, ipm.getValue(LogicalPlan.IS_CONNECTED_TO_DELAY_OPERATOR));
      oper.inputs.add(input);
    }
  }
}
 

public StreamMeta addSinks(String id, Operator.InputPort<?>... sinks)
{
  StreamMeta sm = logicalPlan.getStream(id);
  if (sm == null) {
    throw new AssertionError("Stream " + id + " is not found!");
  }
  for (Operator.InputPort<?> sink : sinks) {
    sm.addSink(sink);
    if (physicalPlan != null) {
      for (InputPortMeta ipm : sm.getSinks()) {
        if (ipm.getPort() == sink) {
          physicalPlan.connectInput(ipm);
        }
      }
    }
  }
  return sm;
}
 

/**
 * Remove the given partition with any associated parallel partitions and
 * per-partition outputStreams.
 *
 * @param oper
 * @return
 */
private void removePartition(PTOperator oper, PMapping operatorMapping)
{
  // remove any parallel partition
  for (PTOutput out : oper.outputs) {
    // copy list as it is modified by recursive remove
    for (PTInput in : Lists.newArrayList(out.sinks)) {
      for (LogicalPlan.InputPortMeta im : in.logicalStream.getSinks()) {
        PMapping m = this.logicalToPTOperator.get(im.getOperatorMeta());
        if (m.parallelPartitions == operatorMapping.parallelPartitions) {
          // associated operator parallel partitioned
          removePartition(in.target, operatorMapping);
          m.partitions.remove(in.target);
        }
      }
    }
  }
  // remove the operator
  removePTOperator(oper);
}
 

/**
 * Remove the given partition with any associated parallel partitions and
 * per-partition outputStreams.
 *
 * @param oper
 * @return
 */
private void removePartition(PTOperator oper, PMapping operatorMapping)
{
  // remove any parallel partition
  for (PTOutput out : oper.outputs) {
    // copy list as it is modified by recursive remove
    for (PTInput in : Lists.newArrayList(out.sinks)) {
      for (LogicalPlan.InputPortMeta im : in.logicalStream.getSinks()) {
        PMapping m = this.logicalToPTOperator.get(im.getOperatorMeta());
        if (m.parallelPartitions == operatorMapping.parallelPartitions) {
          // associated operator parallel partitioned
          removePartition(in.target, operatorMapping);
          m.partitions.remove(in.target);
        }
      }
    }
  }
  // remove the operator
  removePTOperator(oper);
}
 

private Set<PTOperator> getDependentPersistOperators(Collection<PTOperator> operators)
{
  Set<PTOperator> persistOperators = new LinkedHashSet<>();
  if (operators != null) {
    for (PTOperator operator : operators) {
      for (PTInput in : operator.inputs) {
        if (in.logicalStream.getPersistOperator() != null) {
          for (InputPortMeta inputPort : in.logicalStream.getSinksToPersist()) {
            if (inputPort.getOperatorMeta().equals(operator.operatorMeta)) {
              // Redeploy the stream wide persist operator only if the current sink is being persisted
              persistOperators.addAll(getOperators(in.logicalStream.getPersistOperator()));
              break;
            }
          }
        }
        for (Map.Entry<InputPortMeta, OperatorMeta> entry : in.logicalStream.sinkSpecificPersistOperatorMap.entrySet()) {
          // Redeploy sink specific persist operators
          persistOperators.addAll(getOperators(entry.getValue()));
        }
      }
    }
  }
  return persistOperators;
}
 

private Set<PTOperator> getDependentPersistOperators(Collection<PTOperator> operators)
{
  Set<PTOperator> persistOperators = new LinkedHashSet<>();
  if (operators != null) {
    for (PTOperator operator : operators) {
      for (PTInput in : operator.inputs) {
        if (in.logicalStream.getPersistOperator() != null) {
          for (InputPortMeta inputPort : in.logicalStream.getSinksToPersist()) {
            if (inputPort.getOperatorMeta().equals(operator.operatorMeta)) {
              // Redeploy the stream wide persist operator only if the current sink is being persisted
              persistOperators.addAll(getOperators(in.logicalStream.getPersistOperator()));
              break;
            }
          }
        }
        for (Map.Entry<InputPortMeta, OperatorMeta> entry : in.logicalStream.sinkSpecificPersistOperatorMap.entrySet()) {
          // Redeploy sink specific persist operators
          persistOperators.addAll(getOperators(entry.getValue()));
        }
      }
    }
  }
  return persistOperators;
}
 

public StreamMeta addSinks(String id, Operator.InputPort<?>... sinks)
{
  StreamMeta sm = logicalPlan.getStream(id);
  if (sm == null) {
    throw new AssertionError("Stream " + id + " is not found!");
  }
  for (Operator.InputPort<?> sink : sinks) {
    sm.addSink(sink);
    if (physicalPlan != null) {
      for (InputPortMeta ipm : sm.getSinks()) {
        if (ipm.getPort() == sink) {
          physicalPlan.connectInput(ipm);
        }
      }
    }
  }
  return sm;
}
 

public Map<InputPort<?>, PartitionKeys> getPartitionKeys()
{
  Map<InputPort<?>, PartitionKeys> pkeys = null;
  if (partitionKeys != null) {
    pkeys = Maps.newHashMapWithExpectedSize(partitionKeys.size());
    for (Map.Entry<InputPortMeta, PartitionKeys> e : partitionKeys.entrySet()) {
      pkeys.put(e.getKey().getPort(), e.getValue());
    }
  }
  return pkeys;
}
 

private void updatePersistOperatorWithSinkPartitions(InputPortMeta persistInputPort, OperatorMeta persistOperatorMeta, StreamCodecWrapperForPersistance<?> persistCodec, InputPortMeta sinkPortMeta)
{
  Collection<PTOperator> ptOperators = getOperators(sinkPortMeta.getOperatorMeta());
  Collection<PartitionKeys> partitionKeysList = new ArrayList<>();
  for (PTOperator p : ptOperators) {
    PartitionKeys keys = p.partitionKeys.get(sinkPortMeta);
    partitionKeysList.add(keys);
  }

  persistCodec.inputPortToPartitionMap.put(sinkPortMeta, partitionKeysList);
}
 

private List<InputPort<?>> getInputPortList(LogicalPlan.OperatorMeta operatorMeta)
{
  List<InputPort<?>> inputPortList = Lists.newArrayList();

  for (InputPortMeta inputPortMeta: operatorMeta.getInputStreams().keySet()) {
    inputPortList.add(inputPortMeta.getPort());
  }

  return inputPortList;
}
 

public static InputPortMeta getIdentifyingInputPortMeta(PTOperator.PTInput input)
{
  InputPortMeta inputPortMeta;
  PTOperator inputTarget = input.target;
  StreamMeta streamMeta = input.logicalStream;
  if (!inputTarget.isUnifier()) {
    inputPortMeta = getInputPortMeta(inputTarget.getOperatorMeta(), streamMeta);
  } else {
    PTOperator destTarget = getIdentifyingOperator(inputTarget);
    inputPortMeta = getInputPortMeta(destTarget.getOperatorMeta(), streamMeta);
  }
  return inputPortMeta;
}
 

private void printTopology(OperatorMeta operator, DAG tplg, int level)
{
  String prefix = "";
  if (level > 0) {
    prefix = StringUtils.repeat(" ", 20 * (level - 1)) + "   |" + StringUtils.repeat("-", 17);
  }
  logger.debug(prefix + operator.getName());
  for (StreamMeta downStream : operator.getOutputStreams().values()) {
    if (!downStream.getSinks().isEmpty()) {
      for (LogicalPlan.InputPortMeta targetNode : downStream.getSinks()) {
        printTopology(targetNode.getOperatorMeta(), tplg, level + 1);
      }
    }
  }
}
 

@Test
public void testTupleClassAttr() throws Exception
{
  String resourcePath = "/schemaTestTopology.json";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    fail("Could not load " + resourcePath);
  }
  StringWriter writer = new StringWriter();

  IOUtils.copy(is, writer);
  JSONObject json = new JSONObject(writer.toString());

  Configuration conf = new Configuration(false);

  LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
  dag.validate();

  OperatorMeta operator1 = dag.getOperatorMeta("operator1");
  assertEquals("operator1.classname", SchemaTestOperator.class, operator1.getOperator().getClass());

  StreamMeta input1 = dag.getStream("inputStream");
  assertNotNull(input1);
  for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
    Assert.assertEquals("tuple class name required", TestSchema.class, targetPort.getAttributes().get(PortContext.TUPLE_CLASS));
  }
}
 

private void simpleNameInputPortAssertHelper(Attribute<?> attributeObj, LogicalPlan dag, String operatorName, Object queueCapacity, boolean set)
{
  OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);

  for (InputPortMeta inputPortMeta: operatorMeta.getInputStreams().keySet()) {
    if (set) {
      Assert.assertEquals(queueCapacity, inputPortMeta.getValue(attributeObj));
    } else {
      Assert.assertNotEquals(queueCapacity, inputPortMeta.getValue(attributeObj));
    }
  }
}
 

public static InputPortMeta getIdentifyingInputPortMeta(PTOperator.PTInput input)
{
  InputPortMeta inputPortMeta;
  PTOperator inputTarget = input.target;
  StreamMeta streamMeta = input.logicalStream;
  if (!inputTarget.isUnifier()) {
    inputPortMeta = getInputPortMeta(inputTarget.getOperatorMeta(), streamMeta);
  } else {
    PTOperator destTarget = getIdentifyingOperator(inputTarget);
    inputPortMeta = getInputPortMeta(destTarget.getOperatorMeta(), streamMeta);
  }
  return inputPortMeta;
}
 

public static InputPortMeta getInputPortMeta(LogicalPlan.OperatorMeta operatorMeta, StreamMeta streamMeta)
{
  InputPortMeta inputPortMeta = null;
  Map<InputPortMeta, StreamMeta> inputStreams = operatorMeta.getInputStreams();
  for (Map.Entry<InputPortMeta, StreamMeta> entry : inputStreams.entrySet()) {
    if (entry.getValue() == streamMeta) {
      inputPortMeta = entry.getKey();
      break;
    }
  }
  return inputPortMeta;
}
 

public Map<InputPort<?>, PartitionKeys> getPartitionKeys()
{
  Map<InputPort<?>, PartitionKeys> pkeys = null;
  if (partitionKeys != null) {
    pkeys = Maps.newHashMapWithExpectedSize(partitionKeys.size());
    for (Map.Entry<InputPortMeta, PartitionKeys> e : partitionKeys.entrySet()) {
      pkeys.put(e.getKey().getPort(), e.getValue());
    }
  }
  return pkeys;
}
 

public static InputPortMeta getInputPortMeta(LogicalPlan.OperatorMeta operatorMeta, StreamMeta streamMeta)
{
  InputPortMeta inputPortMeta = null;
  Map<InputPortMeta, StreamMeta> inputStreams = operatorMeta.getInputStreams();
  for (Map.Entry<InputPortMeta, StreamMeta> entry : inputStreams.entrySet()) {
    if (entry.getValue() == streamMeta) {
      inputPortMeta = entry.getKey();
      break;
    }
  }
  return inputPortMeta;
}
 

private List<InputPort<?>> getInputPortList(LogicalPlan.OperatorMeta operatorMeta)
{
  List<InputPort<?>> inputPortList = Lists.newArrayList();

  for (InputPortMeta inputPortMeta: operatorMeta.getInputStreams().keySet()) {
    inputPortList.add(inputPortMeta.getPort());
  }

  return inputPortList;
}
 

private void updatePersistOperatorWithSinkPartitions(InputPortMeta persistInputPort, OperatorMeta persistOperatorMeta, StreamCodecWrapperForPersistance<?> persistCodec, InputPortMeta sinkPortMeta)
{
  Collection<PTOperator> ptOperators = getOperators(sinkPortMeta.getOperatorMeta());
  Collection<PartitionKeys> partitionKeysList = new ArrayList<>();
  for (PTOperator p : ptOperators) {
    PartitionKeys keys = p.partitionKeys.get(sinkPortMeta);
    partitionKeysList.add(keys);
  }

  persistCodec.inputPortToPartitionMap.put(sinkPortMeta, partitionKeysList);
}
 

@Test
public void testAddOperator()
{
  LogicalPlan dag = new LogicalPlan();
  TestOperator<String, Integer> firstOperator = new TestOperator<>();
  TestOperator<Integer, Date> secondOperator = new TestOperator<>();
  new ApexStreamImpl<String>().addOperator(firstOperator, null, firstOperator.output, name("first"))
      .endWith(secondOperator, secondOperator.input, name("second"))
      .with(DAG.Locality.THREAD_LOCAL)
      .with(Context.OperatorContext.AUTO_RECORD, true)
      .with("prop", "TestProp").populateDag(dag);
  Assert.assertTrue(dag.getAllOperators().size() == 2);
  Set<String> opNames = new HashSet<>();
  opNames.add("first");
  opNames.add("second");
  for (LogicalPlan.OperatorMeta operatorMeta : dag.getAllOperators()) {
    Assert.assertTrue(operatorMeta.getOperator() instanceof TestOperator);
    Assert.assertTrue(opNames.contains(operatorMeta.getName()));
    if (operatorMeta.getName().equals("second")) {
      // Assert the Context.OperatorContext.AUTO_RECORD attribute has been set to true for second operator
      Assert.assertTrue(operatorMeta.getAttributes().get(Context.OperatorContext.AUTO_RECORD));
      // Assert the prop has been set to TestProp for second operator
      Assert.assertEquals("TestProp", ((TestOperator)operatorMeta.getOperator()).prop);
    } else {
      // Assert the Context.OperatorContext.AUTO_RECORD attribute keeps as default false for first operator
      Assert.assertNull(operatorMeta.getAttributes().get(Context.OperatorContext.AUTO_RECORD));
      // Assert the prop has not been set for first operator
      Assert.assertNull(((TestOperator)operatorMeta.getOperator()).prop);
    }
  }

  Collection<LogicalPlan.StreamMeta> streams = dag.getAllStreams();
  // Assert there is only one stream
  Assert.assertTrue(streams.size() == 1);
  for (LogicalPlan.StreamMeta stream : streams) {

    // Assert the stream is from first operator to second operator
    Assert.assertEquals("first", stream.getSource().getOperatorMeta().getName());
    Collection<InputPortMeta> portMetaCollection = stream.getSinks();
    Assert.assertTrue(1 == portMetaCollection.size());
    for (InputPortMeta inputPortMeta : portMetaCollection) {
      Assert.assertEquals("second", inputPortMeta.getOperatorMeta().getName());
    }

    // Assert the stream is thread local
    Assert.assertTrue(stream.getLocality() == DAG.Locality.THREAD_LOCAL);
  }

}
 

public StreamCodecWrapperForPersistance(Map<InputPortMeta, StreamCodec<Object>> inputStreamCodecs, StreamCodec<Object> specifiedStreamCodec)
{
  this.codecsToMerge = inputStreamCodecs;
  this.setSpecifiedStreamCodec(specifiedStreamCodec);
  inputPortToPartitionMap = new HashMap<>();
}
 

@Test
public void testDynamicPartitioning() throws ClassNotFoundException, IOException
{
  AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());

  final TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
  dag.setOperatorAttribute(console, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TestReceiverOperator>(2));
  dag.setOperatorAttribute(console, Context.OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener)new PartitioningTest.PartitionLoadWatch()));

  final PartitionedTestPersistanceOperator console1 = new PartitionedTestPersistanceOperator();

  StreamMeta s = dag.addStream("Stream1", ascend.outputPort, console.inport);
  dag.setInputPortAttribute(console.inport, PortContext.STREAM_CODEC, new TestPartitionCodec());
  s.persistUsing("persister", console1, console1.inport);

  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
  StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);

  StreamingContainerManager dnm = new StreamingContainerManager(dag);
  PhysicalPlan plan = dnm.getPhysicalPlan();

  List<PTContainer> containers = plan.getContainers();
  Assert.assertEquals("number containers", 4, containers.size());

  for (int i = 0; i < containers.size(); ++i) {
    StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
  }

  LogicalPlan.OperatorMeta passThruMeta = dag.getMeta(console);

  List<PTOperator> ptos = plan.getOperators(passThruMeta);

  PTOperator persistOperatorContainer = null;

  for (PTContainer container : plan.getContainers()) {
    for (PTOperator operator : container.getOperators()) {
      operator.setState(PTOperator.State.ACTIVE);
      if (operator.getName().equals("persister")) {
        persistOperatorContainer = operator;
      }
    }
  }

  // Check that persist operator is part of dependents redeployed
  Set<PTOperator> operators = plan.getDependents(ptos);
  logger.debug("Operators to be re-deployed = {}", operators);
  // Validate that persist operator is part of dependents
  assertTrue("persist operator should be part of the operators to be redeployed", operators.contains(persistOperatorContainer));

  LogicalPlan.StreamMeta s1 = (LogicalPlan.StreamMeta)s;
  StreamCodec codec = s1.getPersistOperatorInputPort().getStreamCodec();

  assertEquals("Codec should be instance of StreamCodecWrapper", codec instanceof StreamCodecWrapperForPersistance, true);
  StreamCodecWrapperForPersistance wrapperCodec = (StreamCodecWrapperForPersistance)codec;

  Entry<InputPortMeta, Collection<PartitionKeys>> keys = (Entry<InputPortMeta, Collection<PartitionKeys>>)wrapperCodec.inputPortToPartitionMap.entrySet().iterator().next();
  logger.debug(keys.toString());
  assertEquals("Size of partitions should be 2", 2, keys.getValue().size());

  for (PTOperator ptOperator : ptos) {
    PartitioningTest.PartitionLoadWatch.put(ptOperator, -1);
    plan.onStatusUpdate(ptOperator);
  }

  dnm.processEvents();

  assertEquals("Input port map", wrapperCodec.inputPortToPartitionMap.size(), 1);

  keys = (Entry<InputPortMeta, Collection<PartitionKeys>>)wrapperCodec.inputPortToPartitionMap.entrySet().iterator().next();
  assertEquals("Size of partitions should be 1 after repartition", 1, keys.getValue().size());
  logger.debug(keys.toString());
}
 

public static PropertiesConfiguration convertToProperties(LogicalPlan dag)
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  Collection<OperatorMeta> allOperators = dag.getAllOperators();

  for (OperatorMeta operatorMeta : allOperators) {
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
    Operator operator = operatorMeta.getOperator();
    props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
    BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
    @SuppressWarnings("rawtypes")
    Iterator entryIterator = operatorProperties.entryIterator();
    while (entryIterator.hasNext()) {
      try {
        @SuppressWarnings("unchecked")
        Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next();
        if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
          props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
        }
      } catch (Exception ex) {
        LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
      }
    }
  }
  Collection<StreamMeta> allStreams = dag.getAllStreams();

  for (StreamMeta streamMeta : allStreams) {
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
    OutputPortMeta source = streamMeta.getSource();
    Collection<InputPortMeta> sinks = streamMeta.getSinks();
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
    String sinksValue = "";
    for (InputPortMeta sink : sinks) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    if (streamMeta.getLocality() != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
    }
  }

  // TBD: Attributes

  return props;
}
 

@Test
public void testLoadFromJson() throws Exception
{
  String resourcePath = "/testTopology.json";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    fail("Could not load " + resourcePath);
  }
  StringWriter writer = new StringWriter();

  IOUtils.copy(is, writer);
  JSONObject json = new JSONObject(writer.toString());

  Configuration conf = new Configuration(false);
  conf.set(StreamingApplication.APEX_PREFIX + "operator.operator3.prop.myStringProperty", "o3StringFromConf");

  LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
  dag.validate();

  assertEquals("DAG attribute CONTAINER_JVM_OPTIONS ", dag.getAttributes().get(DAGContext.CONTAINER_JVM_OPTIONS), "-Xmx16m");
  Map<Class<?>, Class<? extends StringCodec<?>>> stringCodecsMap = Maps.newHashMap();
  stringCodecsMap.put(Integer.class, Integer2String.class);
  assertEquals("DAG attribute STRING_CODECS ", stringCodecsMap, dag.getAttributes().get(DAGContext.STRING_CODECS));
  assertEquals("DAG attribute CONTAINER_OPTS_CONFIGURATOR ", BasicContainerOptConfigurator.class, dag.getAttributes().get(DAGContext.CONTAINER_OPTS_CONFIGURATOR).getClass());

  assertEquals("number of operator confs", 5, dag.getAllOperators().size());
  assertEquals("number of root operators", 1, dag.getRootOperators().size());

  StreamMeta s1 = dag.getStream("n1n2");
  assertNotNull(s1);
  assertTrue("n1n2 inline", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());

  OperatorMeta input = dag.getOperatorMeta("inputOperator");
  TestStatsListener tsl = new TestStatsListener();
  tsl.setIntProp(222);
  List<StatsListener> sll = Lists.<StatsListener>newArrayList(tsl);
  assertEquals("inputOperator STATS_LISTENERS attribute ", sll, input.getAttributes().get(OperatorContext.STATS_LISTENERS));
  for (OutputPortMeta opm : input.getOutputStreams().keySet()) {
    assertTrue("output port of input Operator attribute is JsonStreamCodec ", opm.getAttributes().get(PortContext.STREAM_CODEC) instanceof JsonStreamCodec<?>);
  }

  OperatorMeta operator3 = dag.getOperatorMeta("operator3");
  assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());

  GenericTestOperator doperator3 = (GenericTestOperator)operator3.getOperator();
  assertEquals("myStringProperty " + doperator3, "o3StringFromConf", doperator3.getMyStringProperty());
  assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);

  OperatorMeta operator4 = dag.getOperatorMeta("operator4");
  GenericTestOperator doperator4 = (GenericTestOperator)operator4.getOperator();
  assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
  assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
  assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);

  StreamMeta input1 = dag.getStream("inputStream");
  assertNotNull(input1);
  OperatorMeta inputOperator = dag.getOperatorMeta("inputOperator");
  Assert.assertEquals("input1 source", inputOperator, input1.getSource().getOperatorMeta());
  Set<OperatorMeta> targetNodes = Sets.newHashSet();
  for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
    targetNodes.add(targetPort.getOperatorMeta());
  }
  Assert.assertEquals("operator attribute " + inputOperator, 64, (int)inputOperator.getValue(OperatorContext.MEMORY_MB));
  Assert.assertEquals("port attribute " + inputOperator, 8, (int)input1.getSource().getValue(PortContext.UNIFIER_LIMIT));
  Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
}
 

@Test
public void testLoadFromPropertiesFile() throws IOException
{
  Properties props = new Properties();
  String resourcePath = "/testTopology.properties";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    fail("Could not load " + resourcePath);
  }
  props.load(is);
  LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false)).addFromProperties(props, null);

  LogicalPlan dag = new LogicalPlan();
  pb.populateDAG(dag);
  dag.validate();

  assertEquals("number of operator confs", 5, dag.getAllOperators().size());
  assertEquals("number of root operators", 1, dag.getRootOperators().size());

  StreamMeta s1 = dag.getStream("n1n2");
  assertNotNull(s1);
  assertTrue("n1n2 locality", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());

  OperatorMeta operator3 = dag.getOperatorMeta("operator3");
  assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());

  GenericTestOperator doperator3 = (GenericTestOperator)operator3.getOperator();
  assertEquals("myStringProperty " + doperator3, "myStringPropertyValueFromTemplate", doperator3.getMyStringProperty());
  assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);

  OperatorMeta operator4 = dag.getOperatorMeta("operator4");
  GenericTestOperator doperator4 = (GenericTestOperator)operator4.getOperator();
  assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
  assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
  assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);

  StreamMeta input1 = dag.getStream("inputStream");
  assertNotNull(input1);
  Assert.assertEquals("input1 source", dag.getOperatorMeta("inputOperator"), input1.getSource().getOperatorMeta());
  Set<OperatorMeta> targetNodes = Sets.newHashSet();
  for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
    targetNodes.add(targetPort.getOperatorMeta());
  }

  Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);

}
 

public StreamCodecWrapperForPersistance(Map<InputPortMeta, StreamCodec<Object>> inputStreamCodecs, StreamCodec<Object> specifiedStreamCodec)
{
  this.codecsToMerge = inputStreamCodecs;
  this.setSpecifiedStreamCodec(specifiedStreamCodec);
  inputPortToPartitionMap = new HashMap<>();
}