Java Code Examples for com.datatorrent.stram.plan.logical.LogicalPlan#setOperatorAttribute()

@Test
public void testNumberOfUnifiersWithEvenPartitions()
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
  GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
  GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
  dag.addStream("node1.outport1", node1.outport1, node2.inport1);
  dag.setOperatorAttribute(node1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(8));
  dag.setOutputPortAttribute(node1.outport1, PortContext.UNIFIER_LIMIT, 4);
  PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
  List<PTContainer> containers = plan.getContainers();
  int unifierCount = 0;
  int totalOperators = 0;
  for (PTContainer container : containers) {
    List<PTOperator> operators = container.getOperators();
    for (PTOperator operator : operators) {
      totalOperators++;
      if (operator.isUnifier()) {
        unifierCount++;
      }
    }
  }
  Assert.assertEquals("Number of operators", 12, totalOperators);
  Assert.assertEquals("Number of unifiers", 3, unifierCount);
}
 

@Test
public void testMxNPartitionForSlidingWindow()
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
  GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
  dag.setOperatorAttribute(o1, OperatorContext.APPLICATION_WINDOW_COUNT, 4);
  dag.setOperatorAttribute(o1, OperatorContext.SLIDE_BY_WINDOW_COUNT, 2);
  dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<>(2));
  dag.getOperatorMeta("o1").getMeta(o1.outport1).getUnifierMeta().getAttributes().put(OperatorContext.MEMORY_MB, 1024);
  dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<>(2));
  dag.setOperatorAttribute(o2, OperatorContext.SLIDE_BY_WINDOW_COUNT, 2);
  dag.setOperatorAttribute(o2, OperatorContext.APPLICATION_WINDOW_COUNT, 4);

  dag.addStream("o1.outport1", o1.outport1, o2.inport1);
  dag.addStream("o2.outport1", o2.outport1, o3.inport1);
  PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
  Assert.assertEquals("number of containers", 9, plan.getContainers().size());
}
 

@Test
public void testContainerSizeWithPartitioning()
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
  dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
  dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));

  dag.addStream("o1.outport1", o1.outport1, o2.inport1);
  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 10);
  PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
  Assert.assertEquals("number of containers", 5, plan.getContainers().size());
  PTContainer container;
  for (int i = 0; i < 5; i++) {
    container = plan.getContainers().get(i);
    if (container.getOperators().size() == 1) {
      Assert.assertEquals("container memory is 1536 for container :" + container, 1536, container.getRequiredMemoryMB());
    }
    if (container.getOperators().size() == 2) {
      Assert.assertEquals("container memory is 2048 for container :" + container, 2048, container.getRequiredMemoryMB());
    }
  }
}
 

@Test
public void testNumberOfUnifiers()
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
  GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
  GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
  dag.addStream("node1.outport1", node1.outport1, node2.inport1);
  dag.setOperatorAttribute(node1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(5));
  dag.setOutputPortAttribute(node1.outport1, PortContext.UNIFIER_LIMIT, 3);
  PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
  List<PTContainer> containers = plan.getContainers();
  int unifierCount = 0;
  int totalOperators = 0;
  for (PTContainer container : containers) {
    List<PTOperator> operators = container.getOperators();
    for (PTOperator operator : operators) {
      totalOperators++;
      if (operator.isUnifier()) {
        unifierCount++;
      }
    }
  }
  Assert.assertEquals("Number of operators", 8, totalOperators);
  Assert.assertEquals("Number of unifiers", 2, unifierCount);
}
 

@Test
public void testParallelPartitionForSlidingWindow()
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
  GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
  dag.setOperatorAttribute(o1, OperatorContext.SLIDE_BY_WINDOW_COUNT, 2);
  dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<>(2));
  dag.setInputPortAttribute(o2.inport1, PortContext.PARTITION_PARALLEL, true);
  dag.setOperatorAttribute(o1, OperatorContext.APPLICATION_WINDOW_COUNT, 4);

  dag.addStream("o1.outport1", o1.outport1, o2.inport1);
  dag.addStream("o2.outport1", o2.outport1, o3.inport1);
  PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
  Assert.assertEquals("number of containers", 7, plan.getContainers().size());
}
 

@Test
public void testContainerLocal()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.CONTAINER_LOCAL);
  dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);
  dag.setOperatorAttribute(partitioned,OperatorContext.MEMORY_MB,256);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
  Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size());
  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    csr.container.host = host;
    Assert.assertEquals("Hosts set to host2", "host2", host);
  }
}
 

@Test
public void testUnavailableResources()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.CONTAINER_LOCAL);
  dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);
  dag.setOperatorAttribute(o1, OperatorContext.VCORES, 2);
  dag.setOperatorAttribute(partitioned, OperatorContext.VCORES, 1);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
  Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size());
  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    Assert.assertEquals("number of vcores", 3, csr.container.getRequiredVCores());
    Assert.assertNull("Host is null", host);
  }
}
 

private void test(int applicationWindowCount, int slideByWindowCount) throws Exception
{
  LogicalPlan dag = new LogicalPlan();
  String workingDir = new File("target/sliderTest").getAbsolutePath();
  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
  dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 100);
  Input input = dag.addOperator("Input", new Input());
  Sum sum = dag.addOperator("Sum", new Sum());
  dag.setOperatorAttribute(sum, OperatorContext.APPLICATION_WINDOW_COUNT, applicationWindowCount);
  dag.setOperatorAttribute(sum, OperatorContext.SLIDE_BY_WINDOW_COUNT, slideByWindowCount);
  Validator validate = dag.addOperator("validator", new Validator());
  Validator.numbersValidated = 0;
  validate.numberOfIntegers = applicationWindowCount;
  validate.slideByNumbers = slideByWindowCount;
  dag.addStream("input-sum", input.defaultOutputPort, sum.inputPort);
  dag.addStream("sum-validator", sum.outputPort, validate.validate);
  StramLocalCluster lc = new StramLocalCluster(dag);
  lc.runAsync();

  long startTms = System.currentTimeMillis();
  while (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms) {
    if (validate.numbersValidated > 5) {
      break;
    }
    Thread.sleep(100);
  }
  lc.shutdown();
  Assert.assertTrue("numbers validated more than zero ", validate.numbersValidated > 0);
}
 

@Test
public void testThreadLocal()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.THREAD_LOCAL);
  dag.setOperatorAttribute(o1,OperatorContext.MEMORY_MB,256);
  dag.setOperatorAttribute(partitioned,OperatorContext.MEMORY_MB,256);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
  Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size());
  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    csr.container.host = host;
    Assert.assertEquals("Hosts set to host2", "host2", host);
  }
}
 

@Test
public void testContainerSize()
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
  GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
  dag.setOperatorAttribute(o1,OperatorContext.VCORES,1);
  dag.setOperatorAttribute(o2,OperatorContext.VCORES,2);

  dag.addStream("o1.outport1", o1.outport1, o2.inport1);
  dag.addStream("o2.outport1", o2.outport1, o3.inport1);

  dag.setOperatorAttribute(o2, OperatorContext.MEMORY_MB, 4000);
  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 2);

  PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());

  Assert.assertEquals("number of containers", 2, plan.getContainers().size());
  Assert.assertEquals("memory container 1", 2560, plan.getContainers().get(0).getRequiredMemoryMB());
  Assert.assertEquals("vcores container 1", 1, plan.getContainers().get(0).getRequiredVCores());
  Assert.assertEquals("memory container 2", 4512, plan.getContainers().get(1).getRequiredMemoryMB());
  Assert.assertEquals("vcores container 2", 2, plan.getContainers().get(1).getRequiredVCores());
  Assert.assertEquals("number of operators in container 1", 2, plan.getContainers().get(0).getOperators().size());
}
 

@Test
public void testContainerCores()
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
  GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
  GenericTestOperator o4 = dag.addOperator("o4", GenericTestOperator.class);
  GenericTestOperator o5 = dag.addOperator("o5", GenericTestOperator.class);
  GenericTestOperator o6 = dag.addOperator("o6", GenericTestOperator.class);
  dag.setOperatorAttribute(o1,OperatorContext.VCORES,1);
  dag.setOperatorAttribute(o2,OperatorContext.VCORES,2);
  dag.setOperatorAttribute(o3,OperatorContext.VCORES,3);
  dag.setOperatorAttribute(o4,OperatorContext.VCORES,4);
  dag.setOperatorAttribute(o5,OperatorContext.VCORES,5);
  dag.setOperatorAttribute(o6,OperatorContext.VCORES,6);

  dag.addStream("o1.outport1", o1.outport1, o2.inport1).setLocality(Locality.CONTAINER_LOCAL);
  dag.addStream("o2.outport1", o2.outport1, o3.inport1, o4.inport1).setLocality(Locality.THREAD_LOCAL);
  dag.addStream("o3.output1", o3.outport1, o5.inport1).setLocality(Locality.THREAD_LOCAL);
  dag.addStream("o4.output1", o4.outport1, o5.inport2).setLocality(Locality.THREAD_LOCAL);
  dag.addStream("o5.output1", o5.outport1, o6.inport1).setLocality(Locality.CONTAINER_LOCAL);

  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 2);

  PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());

  Assert.assertEquals("number of containers", 1, plan.getContainers().size());
  Assert.assertEquals("vcores container 1 is 12", 12, plan.getContainers().get(0).getRequiredVCores());
}
 

@Test
public void testNodeLocal()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());


  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  dag.setOperatorAttribute(o1,OperatorContext.MEMORY_MB,256);

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  dag.setOperatorAttribute(partitioned,OperatorContext.MEMORY_MB,256);
  dag.getMeta(partitioned).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");

  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.NODE_LOCAL);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));

  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    csr.container.host = host;
    Assert.assertEquals("Hosts set to host1", "host1", host);
  }

}
 

@Test
public void testContainersForSlidingWindow()
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
  GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
  dag.setOperatorAttribute(o1, OperatorContext.APPLICATION_WINDOW_COUNT, 4);
  dag.setOperatorAttribute(o1, OperatorContext.SLIDE_BY_WINDOW_COUNT, 2);
  dag.getOperatorMeta("o1").getMeta(o1.outport1).getUnifierMeta().getAttributes().put(OperatorContext.MEMORY_MB, 2000);
  dag.getOperatorMeta("o1").getMeta(o1.outport2).getUnifierMeta().getAttributes().put(OperatorContext.MEMORY_MB, 4000);

  dag.addStream("o1.outport1", o1.outport1, o2.inport1);
  dag.addStream("o1.outport2", o1.outport2, o2.inport2);
  dag.addStream("o2.outport1", o2.outport1, o3.inport1);
  PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
  Assert.assertEquals("number of containers", 5, plan.getContainers().size());
  boolean sawOutput1Slider = false;
  boolean sawOutput2Slider = false;
  for (PTContainer container : plan.getContainers()) {
    Assert.assertEquals("number of operators in each container is 1", container.operators.size(), 1);
    if (container.operators.get(0).isUnifier()) {
      String name = container.operators.get(0).getName();
      if (name.equals("o1.outport1#slider")) {
        sawOutput1Slider = true;
        Assert.assertEquals("container memory is 2512", container.getRequiredMemoryMB(), 2512);
      } else if (name.equals("o1.outport2#slider")) {
        sawOutput2Slider = true;
        Assert.assertEquals("container memory is 2512", container.getRequiredMemoryMB(), 4512);
      }
    }
  }
  Assert.assertEquals("Found output1 slider", true, sawOutput1Slider);
  Assert.assertEquals("Found output2 slider", true, sawOutput2Slider);
}
 

@Test
public void validatePositiveOiOiOdiamondWithCores()
{
  logger.info("Checking the logic for sanity checking of OiO");

  LogicalPlan plan = new LogicalPlan();
  ThreadIdValidatingInputOperator inputOperator = plan.addOperator("inputOperator", new ThreadIdValidatingInputOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator1 = plan.addOperator("intermediateOperator1", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator2 = plan.addOperator("intermediateOperator2", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator3 = plan.addOperator("intermediateOperator3", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericIntermediateOperator intermediateOperator4 = plan.addOperator("intermediateOperator4", new ThreadIdValidatingGenericIntermediateOperator());
  ThreadIdValidatingGenericOperatorWithTwoInputPorts outputOperator = plan.addOperator("outputOperator", new ThreadIdValidatingGenericOperatorWithTwoInputPorts());

  plan.addStream("OiOin", inputOperator.output, intermediateOperator1.input, intermediateOperator3.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOIntermediate1", intermediateOperator1.output, intermediateOperator2.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOIntermediate2", intermediateOperator3.output, intermediateOperator4.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOout1", intermediateOperator2.output, outputOperator.input).setLocality(Locality.THREAD_LOCAL);
  plan.addStream("OiOout2", intermediateOperator4.output, outputOperator.input2).setLocality(Locality.THREAD_LOCAL);

  plan.setOperatorAttribute(inputOperator, OperatorContext.VCORES, 1);
  plan.setOperatorAttribute(intermediateOperator1, OperatorContext.VCORES, 1);
  plan.setOperatorAttribute(intermediateOperator2, OperatorContext.VCORES, 2);
  plan.setOperatorAttribute(intermediateOperator3, OperatorContext.VCORES, 3);
  plan.setOperatorAttribute(intermediateOperator4, OperatorContext.VCORES, 5);
  plan.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  try {
    plan.validate();
    Assert.assertTrue("OiOiO extended diamond validation", true);
  } catch (ConstraintViolationException ex) {
    Assert.fail("OIOIO extended diamond validation");
  }
  PhysicalPlan physicalPlan = new PhysicalPlan(plan, new TestPlanContext());
  Assert.assertTrue("number of containers", 1 == physicalPlan.getContainers().size());
  Assert.assertTrue("number of vcores " + physicalPlan.getContainers().get(0).getRequiredVCores(), 5 == physicalPlan.getContainers().get(0).getRequiredVCores());
}
 

@SuppressWarnings("SleepWhileInLoop")
private void baseTestForQueueSize(int maxTuples, TestCollectorStatsListener statsListener, DAG.Locality locality) throws Exception
{
  LogicalPlan dag = new LogicalPlan();
  String workingDir = new File("target/baseTestForQueueSize").getAbsolutePath();
  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
  dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 200);
  TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
  testOper.setMaxTuples(maxTuples);

  TestCollector collector = dag.addOperator("Collector", new TestCollector());
  if (statsListener != null) {
    dag.setOperatorAttribute(collector, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[]{statsListener}));
  }

  dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(locality);

  StramLocalCluster lc = new StramLocalCluster(dag);
  lc.runAsync();
  StreamingContainerManager dnmgr = lc.getStreamingContainerManager();
  Map<Integer, PTOperator> operatorMap = dnmgr.getPhysicalPlan().getAllOperators();
  for (PTOperator p : operatorMap.values()) {
    StramTestSupport.waitForActivation(lc, p);
  }

  long startTms = System.currentTimeMillis();
  if (statsListener != null) {
    while (statsListener.collectorOperatorStats.isEmpty() && (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms)) {
      Thread.sleep(300);
      LOG.debug("Waiting for stats");
    }
  } else {
    while (collector.collectorOperatorStats.isEmpty() && (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms)) {
      Thread.sleep(300);
      LOG.debug("Waiting for stats");
    }
  }
  if (statsListener != null) {
    statsListener.validateStats();
  } else {
    collector.validateStats();
  }
  lc.shutdown();
}
 

@Test
public void testAntiAffinityInOperators()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, testMeta.getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("O1", GenericTestOperator.class);
  dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);

  GenericTestOperator o2 = dag.addOperator("O2", GenericTestOperator.class);
  dag.setOperatorAttribute(o2, OperatorContext.MEMORY_MB, 256);

  dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
  AffinityRulesSet ruleSet = new AffinityRulesSet();

  List<AffinityRule> rules = new ArrayList<>();
  ruleSet.setAffinityRules(rules);
  AffinityRule rule1 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O1", "O2");
  rules.add(rule1);
  dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);

  dag.addStream("o1_outport1", o1.outport1, o2.inport1);// .setLocality(Locality.NODE_LOCAL);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));

  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    csr.container.host = host;
    if (csr.container.getOperators().get(0).getName().equals("O1")) {
      Assert.assertEquals("Hosts set to host1 for Operator O1", "host1", host);
    }
    if (csr.container.getOperators().get(0).getName().equals("O2")) {
      Assert.assertEquals("Hosts set to host2 for Operator O2", "host2", host);
    }
  }
}
 

/**
 * Test partitioning of an input operator (no input port).
 * Cover aspects that are not part of generic operator test.
 * Test scaling from one to multiple partitions with unifier when one partition remains unmodified.
 */
@Test
public void testInputOperatorPartitioning()
{
  LogicalPlan dag = new LogicalPlan();
  final TestInputOperator<Object> o1 = dag.addOperator("o1", new TestInputOperator<>());
  GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
  dag.addStream("o1.outport1", o1.output, o2.inport1);

  OperatorMeta o1Meta = dag.getMeta(o1);
  dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[]{new PartitioningTest.PartitionLoadWatch()}));
  TestPartitioner<TestInputOperator<Object>> partitioner = new TestPartitioner<>();
  dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, partitioner);

  TestPlanContext ctx = new TestPlanContext();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
  PhysicalPlan plan = new PhysicalPlan(dag, ctx);
  Assert.assertEquals("number of containers", 2, plan.getContainers().size());

  List<PTOperator> o1Partitions = plan.getOperators(o1Meta);
  Assert.assertEquals("partitions " + o1Partitions, 1, o1Partitions.size());
  PTOperator o1p1 = o1Partitions.get(0);

  // verify load update generates expected events per configuration
  Assert.assertEquals("stats handlers " + o1p1, 1, o1p1.statsListeners.size());
  StatsListener l = o1p1.statsListeners.get(0);
  Assert.assertTrue("stats handlers " + o1p1.statsListeners, l instanceof PartitioningTest.PartitionLoadWatch);

  PartitioningTest.PartitionLoadWatch.put(o1p1, 1);
  plan.onStatusUpdate(o1p1);
  Assert.assertEquals("scale up triggered", 1, ctx.events.size());
  // add another partition, keep existing as is
  partitioner.extraPartitions.add(new DefaultPartition<>(o1));
  Runnable r = ctx.events.remove(0);
  r.run();
  partitioner.extraPartitions.clear();

  o1Partitions = plan.getOperators(o1Meta);
  Assert.assertEquals("operators after scale up", 2, o1Partitions.size());
  Assert.assertEquals("first partition unmodified", o1p1, o1Partitions.get(0));
  Assert.assertEquals("single output", 1, o1p1.getOutputs().size());
  Assert.assertEquals("output to unifier", 1, o1p1.getOutputs().get(0).sinks.size());

  Set<PTOperator> expUndeploy = Sets.newHashSet(plan.getOperators(dag.getMeta(o2)));
  Set<PTOperator> expDeploy = Sets.newHashSet(o1Partitions.get(1));
  expDeploy.addAll(plan.getMergeOperators(dag.getMeta(o1)));
  expDeploy.addAll(expUndeploy);
  expDeploy.add(o1p1.getOutputs().get(0).sinks.get(0).target);

  Assert.assertEquals("undeploy", expUndeploy, ctx.undeploy);
  Assert.assertEquals("deploy", expDeploy, ctx.deploy);

  for (PTOperator p : o1Partitions) {
    Assert.assertEquals("activation window id " + p, Checkpoint.INITIAL_CHECKPOINT, p.recoveryCheckpoint);
    Assert.assertEquals("checkpoints " + p + " " + p.checkpoints, Lists.newArrayList(), p.checkpoints);
    PartitioningTest.PartitionLoadWatch.put(p, -1);
    plan.onStatusUpdate(p);
  }
  ctx.events.remove(0).run();
  Assert.assertEquals("operators after scale down", 1, plan.getOperators(o1Meta).size());
}
 

/**
 * Test unifier gets removed when number partitions drops to 1.
 */
@Test
public void testRepartitioningScaleDownSinglePartition()
{
  LogicalPlan dag = new LogicalPlan();

  TestInputOperator<?> o1 = dag.addOperator("o1", TestInputOperator.class);
  GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);

  dag.addStream("o1.outport1", o1.output, o2.inport1);
  OperatorMeta o1Meta = dag.getMeta(o1);
  dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
  dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[]{new PartitioningTest.PartitionLoadWatch()}));

  TestPlanContext ctx = new TestPlanContext();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
  PhysicalPlan plan = new PhysicalPlan(dag, ctx);

  List<PTOperator> o1Partitions = plan.getOperators(o1Meta);
  Assert.assertEquals("partitions " + o1Partitions, 2, o1Partitions.size());
  PTOperator o1p1 = o1Partitions.get(0);
  PTOperator p1Doper = o1p1.getOutputs().get(0).sinks.get(0).target;
  Assert.assertSame("", p1Doper.getOperatorMeta(), o1Meta.getMeta(o1.output).getUnifierMeta());
  Assert.assertTrue("unifier ", p1Doper.isUnifier());
  Assert.assertEquals("Unifiers " + o1Meta, 1, o1p1.getOutputs().get(0).sinks.size());

  Collection<PTOperator> o1Unifiers = new ArrayList<>(plan.getOperators(dag.getMeta(o2)).get(0).upstreamMerge.values());

  StatsListener l = o1p1.statsListeners.get(0);
  Assert.assertTrue("stats handlers " + o1p1.statsListeners, l instanceof PartitioningTest.PartitionLoadWatch);
  PartitioningTest.PartitionLoadWatch.put(o1p1, -1);
  PartitioningTest.PartitionLoadWatch.put(o1Partitions.get(1), -1);

  plan.onStatusUpdate(o1p1);
  plan.onStatusUpdate(o1Partitions.get(1));
  Assert.assertEquals("partition scaling triggered", 1, ctx.events.size());
  ctx.events.remove(0).run();

  List<PTOperator> o1NewPartitions = plan.getOperators(o1Meta);
  Assert.assertEquals("partitions " + o1NewPartitions, 1, o1NewPartitions.size());

  List<PTOperator> o1NewUnifiers = new ArrayList<>(plan.getOperators(dag.getMeta(o2)).get(0).upstreamMerge.values());

  Assert.assertEquals("unifiers " + o1Meta, 0, o1NewUnifiers.size());
  p1Doper = o1p1.getOutputs().get(0).sinks.get(0).target;
  Assert.assertTrue("", p1Doper.getOperatorMeta() == dag.getMeta(o2));
  Assert.assertFalse("unifier ", p1Doper.isUnifier());

  Assert.assertTrue("removed unifier from deployment " + ctx.undeploy, ctx.undeploy.containsAll(o1Unifiers));
  Assert.assertFalse("removed unifier from deployment " + ctx.deploy, ctx.deploy.containsAll(o1Unifiers));

  // scale up, ensure unifier is setup at activation checkpoint
  setActivationCheckpoint(o1NewPartitions.get(0), 3);
  PartitioningTest.PartitionLoadWatch.put(o1NewPartitions.get(0), 1);
  plan.onStatusUpdate(o1NewPartitions.get(0));
  Assert.assertEquals("partition scaling triggered", 1, ctx.events.size());
  ctx.events.remove(0).run();

  o1NewUnifiers.addAll(plan.getOperators(dag.getMeta(o2)).get(0).upstreamMerge.values());

  Assert.assertEquals("unifiers " + o1Meta, 1, o1NewUnifiers.size());
  Assert.assertEquals("unifier activation checkpoint " + o1Meta, 3, o1NewUnifiers.get(0).recoveryCheckpoint.windowId);
}
 

/**
 * Test covering scenario when only new partitions are added during dynamic partitioning and there
 * are no changes to existing partitions and partition mapping
 */
@Test
public void testAugmentedDynamicPartitioning()
{
  LogicalPlan dag = new LogicalPlan();

  TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
  dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new TestAugmentingPartitioner<TestGeneratorInputOperator>(3));
  dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener)new PartitioningTest.PartitionLoadWatch()));
  OperatorMeta o1Meta = dag.getMeta(o1);

  GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
  OperatorMeta o2Meta = dag.getMeta(o2);

  dag.addStream("o1.outport1", o1.outport, o2.inport1);

  int maxContainers = 10;
  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, maxContainers);

  TestPlanContext ctx = new TestPlanContext();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);

  PhysicalPlan plan = new PhysicalPlan(dag, ctx);
  Assert.assertEquals("number of containers", 4, plan.getContainers().size());

  List<PTOperator> o1ops = plan.getOperators(o1Meta);
  Assert.assertEquals("number of o1 operators", 3, o1ops.size());

  List<PTOperator> o2ops = plan.getOperators(o2Meta);
  Assert.assertEquals("number of o2 operators", 1, o2ops.size());
  Set<PTOperator> expUndeploy = Sets.newLinkedHashSet();
  expUndeploy.addAll(plan.getOperators(o2Meta));
  expUndeploy.add(plan.getOperators(o2Meta).get(0).upstreamMerge.values().iterator().next());

  for (int i = 0; i < 2; ++i) {
    PartitioningTest.PartitionLoadWatch.put(o1ops.get(i), 1);
    plan.onStatusUpdate(o1ops.get(i));
  }

  ctx.backupRequests = 0;
  ctx.events.remove(0).run();

  Assert.assertEquals("number of containers", 6, plan.getContainers().size());

  Assert.assertEquals("undeployed opertors", expUndeploy, ctx.undeploy);
}
 

@Test
public void testSingleFinalUnifierInputOverride()
{
  LogicalPlan dag = new LogicalPlan();

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
  OperatorMeta o1Meta = dag.getMeta(o1);

  GenericTestOperator o2 =  dag.addOperator("o2", GenericTestOperator.class);
  dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
  dag.setInputPortAttribute(o2.inport1, PortContext.UNIFIER_SINGLE_FINAL, true);
  OperatorMeta o2Meta = dag.getMeta(o2);

  dag.addStream("o1.outport1", o1.outport1, o2.inport1);

  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 10);

  TestPlanContext ctx = new TestPlanContext();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);

  PhysicalPlan plan = new PhysicalPlan(dag, ctx);
  Assert.assertEquals("number of containers", 6, plan.getContainers().size());

  Assert.assertEquals("o1 merge unifiers", 1, plan.getMergeOperators(o1Meta).size());

  dag.setOutputPortAttribute(o1.outport1, PortContext.UNIFIER_SINGLE_FINAL, false);
  ctx = new TestPlanContext();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
  plan = new PhysicalPlan(dag, ctx);
  Assert.assertEquals("number of containers", 6, plan.getContainers().size());

  Assert.assertEquals("o1 merge unifiers", 1, plan.getMergeOperators(o1Meta).size());

  dag.setOutputPortAttribute(o1.outport1, PortContext.UNIFIER_SINGLE_FINAL, true);
  dag.setInputPortAttribute(o2.inport1, PortContext.UNIFIER_SINGLE_FINAL, false);
  ctx = new TestPlanContext();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
  plan = new PhysicalPlan(dag, ctx);
  Assert.assertEquals("number of containers", 5, plan.getContainers().size());

  Set<String> expectedNames = Sets.newHashSet(o1Meta.getMeta(o1.outport1).getUnifierMeta().getName(), o2Meta.getName());
  for (int i = 3; i < 5; ++i) {
    PTContainer container = plan.getContainers().get(i);
    Assert.assertEquals("o2 container size", 2, container.getOperators().size());

    Set<String> names = Sets.newHashSet();
    for (PTOperator operator : container.getOperators()) {
      names.add(operator.getOperatorMeta().getName());
    }
    Assert.assertEquals("o2 container operators", expectedNames, names);
  }
}