Java Code Examples for com.datatorrent.stram.plan.logical.LogicalPlan#getOperatorMeta()

public void setAntiAffinityForContainers(LogicalPlan dag, Collection<AffinityRule> affinityRules, Map<PTOperator, PTContainer> operatorContainerMap)
{
  for (AffinityRule rule : affinityRules) {
    if (rule.getOperatorsList() != null && rule.getType() == Type.ANTI_AFFINITY) {
      for (int i = 0; i < rule.getOperatorsList().size() - 1; i++) {
        for (int j = i + 1; j < rule.getOperatorsList().size(); j++) {
          OperatorPair operators = new OperatorPair(rule.getOperatorsList().get(i), rule.getOperatorsList().get(j));

          PMapping firstPMapping = logicalToPTOperator.get(dag.getOperatorMeta(operators.first));
          OperatorMeta opMeta = dag.getOperatorMeta(operators.second);
          PMapping secondMapping = logicalToPTOperator.get(opMeta);
          for (PTOperator firstPtOperator : firstPMapping.partitions) {
            PTContainer firstContainer = operatorContainerMap.get(firstPtOperator);
            for (PTOperator secondPtOperator : secondMapping.partitions) {
              PTContainer secondContainer = operatorContainerMap.get(secondPtOperator);
              if (firstContainer == secondContainer || firstContainer.getStrictAntiPrefs().contains(secondContainer)) {
                continue;
              }
              if (rule.isRelaxLocality()) {
                firstContainer.getPreferredAntiPrefs().add(secondContainer);
                secondContainer.getPreferredAntiPrefs().add(firstContainer);
              } else {
                firstContainer.getStrictAntiPrefs().add(secondContainer);
                secondContainer.getStrictAntiPrefs().add(firstContainer);
              }
            }
          }
        }
      }
    }
  }
}
 

public void setAntiAffinityForContainers(LogicalPlan dag, Collection<AffinityRule> affinityRules, Map<PTOperator, PTContainer> operatorContainerMap)
{
  for (AffinityRule rule : affinityRules) {
    if (rule.getOperatorsList() != null && rule.getType() == Type.ANTI_AFFINITY) {
      for (int i = 0; i < rule.getOperatorsList().size() - 1; i++) {
        for (int j = i + 1; j < rule.getOperatorsList().size(); j++) {
          OperatorPair operators = new OperatorPair(rule.getOperatorsList().get(i), rule.getOperatorsList().get(j));

          PMapping firstPMapping = logicalToPTOperator.get(dag.getOperatorMeta(operators.first));
          OperatorMeta opMeta = dag.getOperatorMeta(operators.second);
          PMapping secondMapping = logicalToPTOperator.get(opMeta);
          for (PTOperator firstPtOperator : firstPMapping.partitions) {
            PTContainer firstContainer = operatorContainerMap.get(firstPtOperator);
            for (PTOperator secondPtOperator : secondMapping.partitions) {
              PTContainer secondContainer = operatorContainerMap.get(secondPtOperator);
              if (firstContainer == secondContainer || firstContainer.getStrictAntiPrefs().contains(secondContainer)) {
                continue;
              }
              if (rule.isRelaxLocality()) {
                firstContainer.getPreferredAntiPrefs().add(secondContainer);
                secondContainer.getPreferredAntiPrefs().add(firstContainer);
              } else {
                firstContainer.getStrictAntiPrefs().add(secondContainer);
                secondContainer.getStrictAntiPrefs().add(firstContainer);
              }
            }
          }
        }
      }
    }
  }
}
 

private void validateOperatorParent(LogicalPlan dag, String operatorName, String parentModuleName)
{
  LogicalPlan.OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);
  if (parentModuleName == null) {
    Assert.assertNull(operatorMeta.getModuleName());
  } else {
    Assert.assertTrue(parentModuleName.equals(operatorMeta.getModuleName()));
  }
}
 

/**
 * Verify attributes populated on DummyOperator from Level1 module
 */
private void validateOperatorAttribute(LogicalPlan dag, String name, int memory)
{
  LogicalPlan.OperatorMeta oMeta = dag.getOperatorMeta(name);
  Attribute.AttributeMap attrs = oMeta.getAttributes();
  Assert.assertEquals((int)attrs.get(OperatorContext.MEMORY_MB), memory);
  Assert.assertEquals("Application window id is 2 ", (int)attrs.get(OperatorContext.APPLICATION_WINDOW_COUNT), 2);
  Assert.assertEquals("Locality host is host1", attrs.get(OperatorContext.LOCALITY_HOST), "host1");
  Assert.assertEquals(attrs.get(OperatorContext.PARTITIONER).getClass(), TestPartitioner.class);
  Assert.assertEquals("Checkpoint window count ", (int)attrs.get(OperatorContext.CHECKPOINT_WINDOW_COUNT), 120);
  Assert.assertEquals("Operator is stateless ", attrs.get(OperatorContext.STATELESS), true);
  Assert.assertEquals("SPIN MILLIS is set to 20 ", (int)attrs.get(OperatorContext.SPIN_MILLIS), 20);

}
 

@Test
  public void testLogicalPlanConfiguration() throws Exception
  {
    TestOperator<String, Map<String, Number>> bean = new InputTestOperator<>();
    bean.map.put("key1", new Structured());
    bean.stringArray = new String[]{"one", "two", "three"};
    bean.stringList = Lists.newArrayList("four", "five");
    bean.props = new Properties();
    bean.props.setProperty("key1", "value1");
    bean.structuredArray = new Structured[]{new Structured()};
    bean.genericArray = new String[]{"s1"};
    bean.structuredArray[0].name = "s1";
    bean.color = Color.BLUE;
    bean.booleanProp = true;
    bean.realName = "abc";

    ObjectMapper mapper = ObjectMapperFactory.getOperatorValueSerializer();
    String s = mapper.writeValueAsString(bean);
//    LOG.debug(new JSONObject(s).toString(2));
    //
    Assert.assertTrue("Shouldn't contain field 'realName' !", !s.contains("realName"));
    Assert.assertTrue("Should contain property 'alias' !", s.contains("alias"));
    Assert.assertTrue("Shouldn't contain property 'getterOnly' !", !s.contains("getterOnly"));
    JSONObject jsonObj = new JSONObject(s);

    // create the json dag representation
    JSONObject jsonPlan = new JSONObject();
    jsonPlan.put("streams", new JSONArray());
    JSONObject jsonOper = new JSONObject();
    jsonOper.put("name", "Test Operator");
    jsonOper.put("class", InputTestOperator.class.getName());
    jsonOper.put("properties", jsonObj);
    jsonPlan.put("operators", new JSONArray(Lists.newArrayList(jsonOper)));

    Configuration conf = new Configuration(false);
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
    // create logical plan from the json
    LogicalPlan lp = lpc.createFromJson(jsonPlan, "jsontest");
    OperatorMeta om = lp.getOperatorMeta("Test Operator");
    Assert.assertTrue(om.getOperator() instanceof InputTestOperator);
    @SuppressWarnings("rawtypes")
    TestOperator beanBack = (InputTestOperator)om.getOperator();

    // The operator deserialized back from json should be same as original operator
    Assert.assertEquals(bean.map, beanBack.map);
    Assert.assertArrayEquals(bean.stringArray, beanBack.stringArray);
    Assert.assertEquals(bean.stringList, beanBack.stringList);
    Assert.assertEquals(bean.props, beanBack.props);
    Assert.assertArrayEquals(bean.structuredArray, beanBack.structuredArray);
    Assert.assertArrayEquals(bean.genericArray, beanBack.genericArray);
    Assert.assertEquals(bean.color, beanBack.color);
    Assert.assertEquals(bean.booleanProp, beanBack.booleanProp);
    Assert.assertEquals(bean.realName, beanBack.realName);
    Assert.assertEquals(bean.getterOnly, beanBack.getterOnly);

  }