Java Code Examples for com.datatorrent.stram.plan.logical.LogicalPlan#OperatorMeta

private static List<PTOperator> assertNumberPartitions(final int count, final StramLocalCluster lc, final LogicalPlan.OperatorMeta ow) throws Exception
{
  WaitCondition c = new WaitCondition()
  {
    @Override
    public boolean isComplete()
    {
      List<PTOperator> operators = lc.getPlanOperators(ow);
      LOG.debug("Number of operators {}, expected number {}", operators.size(), count);
      return (operators.size() == count);
    }

  };
  StramTestSupport.awaitCompletion(c, 10000);
  Assert.assertTrue("Number partitions match " + ow, c.isComplete());
  return lc.getPlanOperators(ow);
}
 

@Test
public void testMxNMultipleStreamCodecs()
{
  GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
  dag.setOperatorAttribute(node1, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
  GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
  dag.setOperatorAttribute(node2, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
  TestStreamCodec serDe = new TestStreamCodec();
  dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);
  GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
  dag.setOperatorAttribute(node3, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
  TestStreamCodec serDe2 = new TestStreamCodec();
  dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, serDe2);

  dag.addStream("n1n2n3", node1.outport1, node2.inport1, node3.inport1);

  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
  StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);

  StreamingContainerManager dnm = new StreamingContainerManager(dag);
  PhysicalPlan plan = dnm.getPhysicalPlan();

  List<PTContainer> containers = plan.getContainers();

  for (int i = 0; i < containers.size(); ++i) {
    StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
  }

  LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
  LogicalPlan.OperatorMeta n2meta = dag.getMeta(node2);
  LogicalPlan.OperatorMeta n3meta = dag.getMeta(node3);

  // Sanity check that physical operators have been allocated for n1meta and n2meta
  Assert.assertEquals("number operators " + n1meta.getName(), 2, plan.getOperators(n1meta).size());
  Assert.assertEquals("number operators " + n2meta.getName(), 3, plan.getOperators(n2meta).size());
  Assert.assertEquals("number operators " + n3meta.getName(), 3, plan.getOperators(n3meta).size());

  checkMxNStreamCodecs(node1, node2, node3, dnm);
}
 

/**
 * Verify attributes populated on DummyOperator from Level1 module
 */
private void validateOperatorAttribute(LogicalPlan dag, String name, int memory)
{
  LogicalPlan.OperatorMeta oMeta = dag.getOperatorMeta(name);
  Attribute.AttributeMap attrs = oMeta.getAttributes();
  Assert.assertEquals((int)attrs.get(OperatorContext.MEMORY_MB), memory);
  Assert.assertEquals("Application window id is 2 ", (int)attrs.get(OperatorContext.APPLICATION_WINDOW_COUNT), 2);
  Assert.assertEquals("Locality host is host1", attrs.get(OperatorContext.LOCALITY_HOST), "host1");
  Assert.assertEquals(attrs.get(OperatorContext.PARTITIONER).getClass(), TestPartitioner.class);
  Assert.assertEquals("Checkpoint window count ", (int)attrs.get(OperatorContext.CHECKPOINT_WINDOW_COUNT), 120);
  Assert.assertEquals("Operator is stateless ", attrs.get(OperatorContext.STATELESS), true);
  Assert.assertEquals("SPIN MILLIS is set to 20 ", (int)attrs.get(OperatorContext.SPIN_MILLIS), 20);

}
 

@Test
public void testNewOperatorRecoveryWindowIds()
{
  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);

  TestPlanContext ctx = new TestPlanContext();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
  PhysicalPlan plan = new PhysicalPlan(dag, ctx);
  ctx.deploy.clear();
  ctx.undeploy.clear();

  LogicalPlan.OperatorMeta o1Meta = dag.getMeta(o1);
  List<PTOperator> o1Partitions = plan.getOperators(o1Meta);
  PhysicalPlanTest.setActivationCheckpoint(o1Partitions.get(0), 10);

  PlanModifier pm = new PlanModifier(plan);
  GenericTestOperator o2 = new GenericTestOperator();
  GenericTestOperator o3 = new GenericTestOperator();
  pm.addOperator("o2", o2);
  pm.addOperator("o3", o3);
  pm.addStream("s1", o1.outport1, o2.inport2);
  pm.addStream("s2", o2.outport1, o3.inport1);

  pm.applyChanges(ctx);

  LogicalPlan.OperatorMeta o2Meta = plan.getLogicalPlan().getMeta(o2);
  List<PTOperator> o2Partitions = plan.getOperators(o2Meta);
  Assert.assertEquals("o2 activation checkpoint " + o2Meta, 10, o2Partitions.get(0).getRecoveryCheckpoint().windowId);

  LogicalPlan.OperatorMeta o3Meta = plan.getLogicalPlan().getMeta(o3);
  List<PTOperator> o3Partitions = plan.getOperators(o3Meta);
  Assert.assertEquals("o3 activation checkpoint " + o2Meta, 10, o3Partitions.get(0).getRecoveryCheckpoint().windowId);
}
 

private void validateOperatorParent(LogicalPlan dag, String operatorName, String parentModuleName)
{
  LogicalPlan.OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);
  if (parentModuleName == null) {
    Assert.assertNull(operatorMeta.getModuleName());
  } else {
    Assert.assertTrue(parentModuleName.equals(operatorMeta.getModuleName()));
  }
}
 

private List<InputPort<?>> getInputPortList(LogicalPlan.OperatorMeta operatorMeta)
{
  List<InputPort<?>> inputPortList = Lists.newArrayList();

  for (InputPortMeta inputPortMeta: operatorMeta.getInputStreams().keySet()) {
    inputPortList.add(inputPortMeta.getPort());
  }

  return inputPortList;
}
 

public OperatorStatus(int operatorId, LogicalPlan.OperatorMeta om)
{
  this.operatorId = operatorId;
  this.operatorMeta = om;
  LogicalPlan dag = om.getDAG();
  throughputCalculationInterval = dag.getValue(LogicalPlan.THROUGHPUT_CALCULATION_INTERVAL);
  throughputCalculationMaxSamples = dag.getValue(LogicalPlan.THROUGHPUT_CALCULATION_MAX_SAMPLES);
  int heartbeatInterval = dag.getValue(LogicalPlan.HEARTBEAT_INTERVAL_MILLIS);

  cpuNanosPMSMA = new TimedMovingAverageLong(throughputCalculationMaxSamples, throughputCalculationInterval);
  latencyMA = new MovingAverageLong(throughputCalculationInterval / heartbeatInterval);
  checkpointTimeMA = new MovingAverageLong(throughputCalculationInterval / heartbeatInterval);
  this.windowProcessingTimeoutMillis = dag.getValue(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS)
    * om.getValue(OperatorContext.TIMEOUT_WINDOW_COUNT);
}
 

private JSONObject getMetricsSchemaData(LogicalPlan.OperatorMeta operatorMeta, Map<String, Object> aggregates)
{
  JSONObject result = new JSONObject();
  try {
    result.put("type", METRICS_SCHEMA);
    result.put("version", METRICS_SCHEMA_VERSION);
    result.put("appUser", appContext.getUser());
    result.put("appName", dnmgr.getApplicationAttributes().get(DAGContext.APPLICATION_NAME));
    result.put("logicalOperatorName", operatorMeta.getName());

    MetricAggregatorMeta metricAggregatorMeta = operatorMeta.getMetricAggregatorMeta();
    JSONArray valueSchemas = new JSONArray();
    for (Map.Entry<String, Object> entry : aggregates.entrySet()) {
      String metricName = entry.getKey();
      Object metricValue = entry.getValue();
      JSONObject valueSchema = new JSONObject();
      valueSchema.put("name", metricName);
      Class<?> type = ClassUtils.wrapperToPrimitive(metricValue.getClass());
      valueSchema.put("type", type == null ? metricValue.getClass().getCanonicalName() : type);
      String[] dimensionAggregators = metricAggregatorMeta.getDimensionAggregatorsFor(metricName);
      if (dimensionAggregators != null) {
        valueSchema.put("dimensionAggregators", Arrays.asList(dimensionAggregators));
      }
      valueSchemas.put(valueSchema);
    }
    result.put("values", valueSchemas);
    String[] timeBuckets = metricAggregatorMeta.getTimeBuckets();
    if (timeBuckets != null) {
      result.put("timeBuckets", Arrays.asList(timeBuckets));
    }

  } catch (JSONException ex) {
    throw new RuntimeException(ex);
  }
  return result;
}
 

public static InputPortMeta getInputPortMeta(LogicalPlan.OperatorMeta operatorMeta, StreamMeta streamMeta)
{
  InputPortMeta inputPortMeta = null;
  Map<InputPortMeta, StreamMeta> inputStreams = operatorMeta.getInputStreams();
  for (Map.Entry<InputPortMeta, StreamMeta> entry : inputStreams.entrySet()) {
    if (entry.getValue() == streamMeta) {
      inputPortMeta = entry.getKey();
      break;
    }
  }
  return inputPortMeta;
}
 

public OperatorStatus(int operatorId, LogicalPlan.OperatorMeta om)
{
  this.operatorId = operatorId;
  this.operatorMeta = om;
  LogicalPlan dag = om.getDAG();
  throughputCalculationInterval = dag.getValue(LogicalPlan.THROUGHPUT_CALCULATION_INTERVAL);
  throughputCalculationMaxSamples = dag.getValue(LogicalPlan.THROUGHPUT_CALCULATION_MAX_SAMPLES);
  int heartbeatInterval = dag.getValue(LogicalPlan.HEARTBEAT_INTERVAL_MILLIS);

  cpuNanosPMSMA = new TimedMovingAverageLong(throughputCalculationMaxSamples, throughputCalculationInterval);
  latencyMA = new MovingAverageLong(throughputCalculationInterval / heartbeatInterval);
  checkpointTimeMA = new MovingAverageLong(throughputCalculationInterval / heartbeatInterval);
  this.windowProcessingTimeoutMillis = dag.getValue(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS)
    * om.getValue(OperatorContext.TIMEOUT_WINDOW_COUNT);
}
 

public void pushMetricsSchema(LogicalPlan.OperatorMeta operatorMeta, Map<String, Object> aggregates) throws IOException
{
  JSONObject schema = operatorSchemas.get(operatorMeta.getName());
  if (schema == null) {
    schema = getMetricsSchemaData(operatorMeta, aggregates);
    operatorSchemas.put(operatorMeta.getName(), schema);
  }
  metricsTransport.push(schema.toString());
}
 

private JSONObject getMetricsSchemaData(LogicalPlan.OperatorMeta operatorMeta, Map<String, Object> aggregates)
{
  JSONObject result = new JSONObject();
  try {
    result.put("type", METRICS_SCHEMA);
    result.put("version", METRICS_SCHEMA_VERSION);
    result.put("appUser", appContext.getUser());
    result.put("appName", dnmgr.getApplicationAttributes().get(DAGContext.APPLICATION_NAME));
    result.put("logicalOperatorName", operatorMeta.getName());

    MetricAggregatorMeta metricAggregatorMeta = operatorMeta.getMetricAggregatorMeta();
    JSONArray valueSchemas = new JSONArray();
    for (Map.Entry<String, Object> entry : aggregates.entrySet()) {
      String metricName = entry.getKey();
      Object metricValue = entry.getValue();
      JSONObject valueSchema = new JSONObject();
      valueSchema.put("name", metricName);
      Class<?> type = ClassUtils.wrapperToPrimitive(metricValue.getClass());
      valueSchema.put("type", type == null ? metricValue.getClass().getCanonicalName() : type);
      String[] dimensionAggregators = metricAggregatorMeta.getDimensionAggregatorsFor(metricName);
      if (dimensionAggregators != null) {
        valueSchema.put("dimensionAggregators", Arrays.asList(dimensionAggregators));
      }
      valueSchemas.put(valueSchema);
    }
    result.put("values", valueSchemas);
    String[] timeBuckets = metricAggregatorMeta.getTimeBuckets();
    if (timeBuckets != null) {
      result.put("timeBuckets", Arrays.asList(timeBuckets));
    }

  } catch (JSONException ex) {
    throw new RuntimeException(ex);
  }
  return result;
}
 

private void validateTopLevelOperators(LogicalPlan dag)
{
  List<String> operatorNames = new ArrayList<>();
  for (LogicalPlan.OperatorMeta operatorMeta : dag.getAllOperators()) {
    operatorNames.add(operatorMeta.getName());
  }
  Assert.assertTrue(operatorNames.contains("O1"));
  Assert.assertTrue(operatorNames.contains("O2"));
  Assert.assertTrue(operatorNames.contains(componentName("Ma", "M1", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Ma", "M2", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Ma", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Mb", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Mb", "M1", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Mb", "O2")));
  Assert.assertTrue(operatorNames.contains(componentName("Mc", "M1", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Mc", "M2", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Mc", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Md", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Md", "M1", "O1")));
  Assert.assertTrue(operatorNames.contains(componentName("Md", "O2")));

  validateOperatorPropertyValue(dag, "O1", 1);
  validateOperatorPropertyValue(dag, "O2", 2);
  validateOperatorPropertyValue(dag, componentName("Ma", "M1", "O1"), 11);
  validateOperatorPropertyValue(dag, componentName("Ma", "M2", "O1"), 12);
  validateOperatorPropertyValue(dag, componentName("Ma", "O1"), 13);
  validateOperatorPropertyValue(dag, componentName("Mb", "O1"), 21);
  validateOperatorPropertyValue(dag, componentName("Mb", "M1", "O1"), 22);
  validateOperatorPropertyValue(dag, componentName("Mb", "O2"), 23);
  validateOperatorPropertyValue(dag, componentName("Mc", "M1", "O1"), 31);
  validateOperatorPropertyValue(dag, componentName("Mc", "M2", "O1"), 32);
  validateOperatorPropertyValue(dag, componentName("Mc", "O1"), 33);
  validateOperatorPropertyValue(dag, componentName("Md", "O1"), 41);
  validateOperatorPropertyValue(dag, componentName("Md", "M1", "O1"), 42);
  validateOperatorPropertyValue(dag, componentName("Md", "O2"), 43);

  validateOperatorParent(dag, "O1", null);
  validateOperatorParent(dag, "O2", null);
  validateOperatorParent(dag, componentName("Ma", "M1", "O1"), componentName("Ma", "M1"));
  validateOperatorParent(dag, componentName("Ma", "M2", "O1"), componentName("Ma", "M2"));
  validateOperatorParent(dag, componentName("Ma", "O1"), "Ma");
  validateOperatorParent(dag, componentName("Mb", "O1"), "Mb");
  validateOperatorParent(dag, componentName("Mb", "M1", "O1"), componentName("Mb", "M1"));
  validateOperatorParent(dag, componentName("Mb", "O2"), "Mb");
  validateOperatorParent(dag, componentName("Mc", "M1", "O1"), componentName("Mc", "M1"));
  validateOperatorParent(dag, componentName("Mc", "M2", "O1"), componentName("Mc", "M2"));
  validateOperatorParent(dag, componentName("Mc", "O1"), "Mc");
  validateOperatorParent(dag, componentName("Md", "O1"), "Md");
  validateOperatorParent(dag, componentName("Md", "M1", "O1"), componentName("Md", "M1"));
  validateOperatorParent(dag, componentName("Md", "O2"), "Md");

  validateOperatorAttribute(dag, componentName("Ma", "M1", "O1"), 1024);
  validateOperatorAttribute(dag, componentName("Ma", "M2", "O1"), 2048);
  validateOperatorAttribute(dag, componentName("Mb", "M1", "O1"), 4096);
  validateOperatorAttribute(dag, componentName("Mc", "M1", "O1"), 1024);
  validateOperatorAttribute(dag, componentName("Mc", "M2", "O1"), 2048);

  validatePortAttribute(dag, componentName("Ma", "M1", "O1"), 1);
  validatePortAttribute(dag, componentName("Ma", "M2", "O1"), 2);
  validatePortAttribute(dag, componentName("Mb", "M1", "O1"), 3);
  validatePortAttribute(dag, componentName("Mc", "M1", "O1"), 1);
  validatePortAttribute(dag, componentName("Mc", "M2", "O1"), 2);
}
 

@Test
public void testAddOperator()
{
  LogicalPlan dag = new LogicalPlan();
  TestOperator<String, Integer> firstOperator = new TestOperator<>();
  TestOperator<Integer, Date> secondOperator = new TestOperator<>();
  new ApexStreamImpl<String>().addOperator(firstOperator, null, firstOperator.output, name("first"))
      .endWith(secondOperator, secondOperator.input, name("second"))
      .with(DAG.Locality.THREAD_LOCAL)
      .with(Context.OperatorContext.AUTO_RECORD, true)
      .with("prop", "TestProp").populateDag(dag);
  Assert.assertTrue(dag.getAllOperators().size() == 2);
  Set<String> opNames = new HashSet<>();
  opNames.add("first");
  opNames.add("second");
  for (LogicalPlan.OperatorMeta operatorMeta : dag.getAllOperators()) {
    Assert.assertTrue(operatorMeta.getOperator() instanceof TestOperator);
    Assert.assertTrue(opNames.contains(operatorMeta.getName()));
    if (operatorMeta.getName().equals("second")) {
      // Assert the Context.OperatorContext.AUTO_RECORD attribute has been set to true for second operator
      Assert.assertTrue(operatorMeta.getAttributes().get(Context.OperatorContext.AUTO_RECORD));
      // Assert the prop has been set to TestProp for second operator
      Assert.assertEquals("TestProp", ((TestOperator)operatorMeta.getOperator()).prop);
    } else {
      // Assert the Context.OperatorContext.AUTO_RECORD attribute keeps as default false for first operator
      Assert.assertNull(operatorMeta.getAttributes().get(Context.OperatorContext.AUTO_RECORD));
      // Assert the prop has not been set for first operator
      Assert.assertNull(((TestOperator)operatorMeta.getOperator()).prop);
    }
  }

  Collection<LogicalPlan.StreamMeta> streams = dag.getAllStreams();
  // Assert there is only one stream
  Assert.assertTrue(streams.size() == 1);
  for (LogicalPlan.StreamMeta stream : streams) {

    // Assert the stream is from first operator to second operator
    Assert.assertEquals("first", stream.getSource().getOperatorMeta().getName());
    Collection<InputPortMeta> portMetaCollection = stream.getSinks();
    Assert.assertTrue(1 == portMetaCollection.size());
    for (InputPortMeta inputPortMeta : portMetaCollection) {
      Assert.assertEquals("second", inputPortMeta.getOperatorMeta().getName());
    }

    // Assert the stream is thread local
    Assert.assertTrue(stream.getLocality() == DAG.Locality.THREAD_LOCAL);
  }

}
 

@Test
public void testInjectionOfDefaultMetricsAggregator() throws Exception
{
  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());

  TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);

  OperatorWithMetricMethod o1 = dag.addOperator("o1", OperatorWithMetricMethod.class);

  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  dag.addStream("TestTuples", inputOperator.outport, o1.inport1);

  lpc.prepareDAG(dag, null, "AutoMetricTest");

  LogicalPlan.OperatorMeta o1meta = dag.getOperatorMeta("o1");
  Assert.assertNotNull("default aggregator injected", o1meta.getMetricAggregatorMeta().getAggregator());
}
 

@Test
public void testStreamCodec()
{
  GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
  GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
  GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
  dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, new TestStreamCodec());

  dag.addStream("n1n2", node1.outport1, node2.inport1);
  dag.addStream("n2n3", node2.outport1, node3.inport1);

  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
  StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);

  StreamingContainerManager dnm = new StreamingContainerManager(dag);
  PhysicalPlan plan = dnm.getPhysicalPlan();

  List<PTContainer> containers = plan.getContainers();
  Assert.assertEquals("number containers", 3, containers.size());

  for (int i = 0; i < containers.size(); ++i) {
    StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
  }

  LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
  LogicalPlan.OperatorMeta n2meta = dag.getMeta(node2);
  LogicalPlan.OperatorMeta n3meta = dag.getMeta(node3);

  OperatorDeployInfo n1di = getSingleOperatorDeployInfo(node1, dnm);

  OperatorDeployInfo.OutputDeployInfo n1odi = getOutputDeployInfo(n1di, n1meta.getMeta(node1.outport1));
  String id = n1meta.getName() + " " + n1odi.portName;
  Assert.assertEquals("number stream codecs " + id, n1odi.streamCodecs.size(), 1);
  Assert.assertTrue("No user set stream codec", n1odi.streamCodecs.containsValue(null));

  OperatorDeployInfo n2di = getSingleOperatorDeployInfo(node2, dnm);

  OperatorDeployInfo.InputDeployInfo n2idi = getInputDeployInfo(n2di, n2meta.getMeta(node2.inport1));
  id = n2meta.getName() + " " + n2idi.portName;
  Assert.assertEquals("number stream codecs " + id, n2idi.streamCodecs.size(), 1);
  Assert.assertTrue("No user set stream codec", n2idi.streamCodecs.containsValue(null));

  OperatorDeployInfo.OutputDeployInfo n2odi = getOutputDeployInfo(n2di, n2meta.getMeta(node2.outport1));
  id = n2meta.getName() + " " + n2odi.portName;
  Assert.assertEquals("number stream codecs " + id, n2odi.streamCodecs.size(), 1);
  checkPresentStreamCodec(n3meta, node3.inport1, n2odi.streamCodecs, id, plan);


  OperatorDeployInfo n3di = getSingleOperatorDeployInfo(node3, dnm);

  OperatorDeployInfo.InputDeployInfo n3idi = getInputDeployInfo(n3di, n3meta.getMeta(node3.inport1));
  id = n3meta.getName() + " " + n3idi.portName;
  Assert.assertEquals("number stream codecs " + id, n3idi.streamCodecs.size(), 1);
  checkPresentStreamCodec(n3meta, node3.inport1, n3idi.streamCodecs, id, plan);
}
 

@Test
public void testDynamicPartitioning() throws ClassNotFoundException, IOException
{
  AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());

  final TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
  dag.setOperatorAttribute(console, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TestReceiverOperator>(2));
  dag.setOperatorAttribute(console, Context.OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener)new PartitioningTest.PartitionLoadWatch()));

  final PartitionedTestPersistanceOperator console1 = new PartitionedTestPersistanceOperator();

  StreamMeta s = dag.addStream("Stream1", ascend.outputPort, console.inport);
  dag.setInputPortAttribute(console.inport, PortContext.STREAM_CODEC, new TestPartitionCodec());
  s.persistUsing("persister", console1, console1.inport);

  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
  StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);

  StreamingContainerManager dnm = new StreamingContainerManager(dag);
  PhysicalPlan plan = dnm.getPhysicalPlan();

  List<PTContainer> containers = plan.getContainers();
  Assert.assertEquals("number containers", 4, containers.size());

  for (int i = 0; i < containers.size(); ++i) {
    StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
  }

  LogicalPlan.OperatorMeta passThruMeta = dag.getMeta(console);

  List<PTOperator> ptos = plan.getOperators(passThruMeta);

  PTOperator persistOperatorContainer = null;

  for (PTContainer container : plan.getContainers()) {
    for (PTOperator operator : container.getOperators()) {
      operator.setState(PTOperator.State.ACTIVE);
      if (operator.getName().equals("persister")) {
        persistOperatorContainer = operator;
      }
    }
  }

  // Check that persist operator is part of dependents redeployed
  Set<PTOperator> operators = plan.getDependents(ptos);
  logger.debug("Operators to be re-deployed = {}", operators);
  // Validate that persist operator is part of dependents
  assertTrue("persist operator should be part of the operators to be redeployed", operators.contains(persistOperatorContainer));

  LogicalPlan.StreamMeta s1 = (LogicalPlan.StreamMeta)s;
  StreamCodec codec = s1.getPersistOperatorInputPort().getStreamCodec();

  assertEquals("Codec should be instance of StreamCodecWrapper", codec instanceof StreamCodecWrapperForPersistance, true);
  StreamCodecWrapperForPersistance wrapperCodec = (StreamCodecWrapperForPersistance)codec;

  Entry<InputPortMeta, Collection<PartitionKeys>> keys = (Entry<InputPortMeta, Collection<PartitionKeys>>)wrapperCodec.inputPortToPartitionMap.entrySet().iterator().next();
  logger.debug(keys.toString());
  assertEquals("Size of partitions should be 2", 2, keys.getValue().size());

  for (PTOperator ptOperator : ptos) {
    PartitioningTest.PartitionLoadWatch.put(ptOperator, -1);
    plan.onStatusUpdate(ptOperator);
  }

  dnm.processEvents();

  assertEquals("Input port map", wrapperCodec.inputPortToPartitionMap.size(), 1);

  keys = (Entry<InputPortMeta, Collection<PartitionKeys>>)wrapperCodec.inputPortToPartitionMap.entrySet().iterator().next();
  assertEquals("Size of partitions should be 1 after repartition", 1, keys.getValue().size());
  logger.debug(keys.toString());
}
 

@Test
public void testPartitioningStreamCodec()
{
  GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
  GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
  dag.setOperatorAttribute(node2, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
  TestStreamCodec serDe = new TestStreamCodec();
  dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);

  dag.addStream("n1n2", node1.outport1, node2.inport1);

  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
  StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);

  StreamingContainerManager dnm = new StreamingContainerManager(dag);
  PhysicalPlan plan = dnm.getPhysicalPlan();

  List<PTContainer> containers = plan.getContainers();
  Assert.assertEquals("number containers", 4, containers.size());

  for (int i = 0; i < containers.size(); ++i) {
    StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
  }

  LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
  LogicalPlan.OperatorMeta n2meta = dag.getMeta(node2);

  OperatorDeployInfo n1di = getSingleOperatorDeployInfo(node1, dnm);

  OperatorDeployInfo.OutputDeployInfo n1odi = getOutputDeployInfo(n1di, n1meta.getMeta(node1.outport1));
  String id = n1meta.getName() + " " + n1odi.portName;
  Assert.assertEquals("number stream codecs " + id, n1odi.streamCodecs.size(), 1);
  checkPresentStreamCodec(n2meta, node2.inport1, n1odi.streamCodecs, id, plan);


  List<PTOperator> operators = plan.getOperators(n2meta);
  Assert.assertEquals("number operators " + n2meta.getName(), 3, operators.size());
  for (PTOperator operator : operators) {
    OperatorDeployInfo odi = getOperatorDeployInfo(operator, n2meta.getName(), dnm);

    OperatorDeployInfo.InputDeployInfo idi = getInputDeployInfo(odi, n2meta.getMeta(node2.inport1));
    id = n2meta.getName() + " " + idi.portName;
    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
    checkPresentStreamCodec(n2meta, node2.inport1, idi.streamCodecs, id, plan);
  }
}
 

@Test
public void testMultipleInputStreamCodec()
{
  GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
  TestStreamCodec serDe = new TestStreamCodec();
  GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
  dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);
  GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
  dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, serDe);

  dag.addStream("n1n2n3", node1.outport1, node2.inport1, node3.inport1);

  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
  StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);

  StreamingContainerManager dnm = new StreamingContainerManager(dag);
  PhysicalPlan plan = dnm.getPhysicalPlan();

  List<PTContainer> containers = plan.getContainers();
  Assert.assertEquals("number containers", 3, containers.size());

  for (int i = 0; i < containers.size(); ++i) {
    StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
  }

  LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
  LogicalPlan.OperatorMeta n2meta = dag.getMeta(node2);
  LogicalPlan.OperatorMeta n3meta = dag.getMeta(node3);

  OperatorDeployInfo n1di = getSingleOperatorDeployInfo(node1, dnm);

  OperatorDeployInfo.OutputDeployInfo n1odi = getOutputDeployInfo(n1di, n1meta.getMeta(node1.outport1));
  String id = n1meta.getName() + " " + n1odi.portName;
  Assert.assertEquals("number stream codecs " + id, n1odi.streamCodecs.size(), 1);
  checkPresentStreamCodec(n2meta, node2.inport1, n1odi.streamCodecs, id, plan);

  OperatorDeployInfo n2di = getSingleOperatorDeployInfo(node2, dnm);

  OperatorDeployInfo.InputDeployInfo n2idi = getInputDeployInfo(n2di, n2meta.getMeta(node2.inport1));
  id = n2meta.getName() + " " + n2idi.portName;
  Assert.assertEquals("number stream codecs " + id, n2idi.streamCodecs.size(), 1);
  checkPresentStreamCodec(n2meta, node2.inport1, n2idi.streamCodecs, id, plan);

  OperatorDeployInfo n3di = getSingleOperatorDeployInfo(node3, dnm);

  OperatorDeployInfo.InputDeployInfo n3idi = getInputDeployInfo(n3di, n3meta.getMeta(node3.inport1));
  id = n3meta.getName() + " " + n3idi.portName;
  Assert.assertEquals("number stream codecs " + id, n3idi.streamCodecs.size(), 1);
  checkPresentStreamCodec(n3meta, node3.inport1, n3idi.streamCodecs, id, plan);
}
 

@Test
public void testMultipleStreamCodecs()
{
  GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
  GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
  GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
  TestStreamCodec serDe = new TestStreamCodec();
  dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);
  TestStreamCodec2 serDe2 = new TestStreamCodec2();
  dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, serDe2);

  dag.addStream("n1n2n3", node1.outport1, node2.inport1, node3.inport1);

  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
  StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);

  StreamingContainerManager dnm = new StreamingContainerManager(dag);
  PhysicalPlan plan = dnm.getPhysicalPlan();

  List<PTContainer> containers = plan.getContainers();
  Assert.assertEquals("number containers", 3, containers.size());

  for (int i = 0; i < containers.size(); ++i) {
    StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
  }

  LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
  LogicalPlan.OperatorMeta n2meta = dag.getMeta(node2);
  LogicalPlan.OperatorMeta n3meta = dag.getMeta(node3);

  OperatorDeployInfo n1di = getSingleOperatorDeployInfo(node1, dnm);

  OperatorDeployInfo.OutputDeployInfo n1odi = getOutputDeployInfo(n1di, n1meta.getMeta(node1.outport1));
  String id = n1meta.getName() + " " + n1odi.portName;
  Assert.assertEquals("number stream codecs " + id, n1odi.streamCodecs.size(), 2);
  checkPresentStreamCodec(n2meta, node2.inport1, n1odi.streamCodecs, id, plan);
  checkPresentStreamCodec(n3meta, node3.inport1, n1odi.streamCodecs, id, plan);

  OperatorDeployInfo n2di = getSingleOperatorDeployInfo(node2, dnm);

  OperatorDeployInfo.InputDeployInfo n2idi = getInputDeployInfo(n2di, n2meta.getMeta(node2.inport1));
  id = n2meta.getName() + " " + n2idi.portName;
  Assert.assertEquals("number stream codecs " + id, n2idi.streamCodecs.size(), 1);
  checkPresentStreamCodec(n2meta, node2.inport1, n2idi.streamCodecs, id, plan);

  OperatorDeployInfo n3di = getSingleOperatorDeployInfo(node3, dnm);

  OperatorDeployInfo.InputDeployInfo n3idi = getInputDeployInfo(n3di, n3meta.getMeta(node3.inport1));
  id = n3meta.getName() + " " + n3idi.portName;
  Assert.assertEquals("number stream codecs " + id, n3idi.streamCodecs.size(), 1);
  checkPresentStreamCodec(n3meta, node3.inport1, n3idi.streamCodecs, id, plan);
}