Java Code Examples for com.datatorrent.stram.plan.logical.LogicalPlanConfiguration#prepareDAG()

The following examples show how to use com.datatorrent.stram.plan.logical.LogicalPlanConfiguration#prepareDAG() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: AutoMetricTest.java From attic-apex-core with Apache License 2.0

6 votes

@Test
public void testMetrics() throws Exception
{
  CountDownLatch latch = new CountDownLatch(1);

  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());

  TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);

  OperatorWithMetrics o1 = dag.addOperator("o1", OperatorWithMetrics.class);
  MockAggregator aggregator = new MockAggregator(latch);
  dag.setOperatorAttribute(o1, Context.OperatorContext.METRICS_AGGREGATOR, aggregator);

  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  dag.addStream("TestTuples", inputOperator.outport, o1.inport1);

  lpc.prepareDAG(dag, null, "AutoMetricTest");

  StramLocalCluster lc = new StramLocalCluster(dag);
  lc.runAsync();
  latch.await();

  Assert.assertEquals("progress", 1L, ((Long)aggregator.result.get("progress")).longValue());
  lc.shutdown();
}

Example 2

Source File: AutoMetricTest.java From attic-apex-core with Apache License 2.0

6 votes

@Test
@Ignore
public void testMetricsAggregations() throws Exception
{
  CountDownLatch latch = new CountDownLatch(2);

  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());

  TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);

  OperatorWithMetrics o1 = dag.addOperator("o1", OperatorWithMetrics.class);
  MockAggregator aggregator = new MockAggregator(latch);
  dag.setOperatorAttribute(o1, Context.OperatorContext.METRICS_AGGREGATOR, aggregator);

  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
  dag.setOperatorAttribute(o1, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TestGeneratorInputOperator>(2));

  dag.addStream("TestTuples", inputOperator.outport, o1.inport1);

  lpc.prepareDAG(dag, null, "AutoMetricTest");
  StramLocalCluster lc = new StramLocalCluster(dag);
  lc.runAsync();
  latch.await();
  Assert.assertEquals("progress", 2L, ((Long)aggregator.result.get("progress")).longValue());
  lc.shutdown();
}

Example 3

Source File: AutoMetricTest.java From attic-apex-core with Apache License 2.0

6 votes

@Test
public void testDefaultMetricsAggregator() throws Exception
{
  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());

  TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);

  CountDownLatch latch = new CountDownLatch(1);
  OperatorAndAggregator o1 = dag.addOperator("o1", new OperatorAndAggregator(latch));

  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  dag.addStream("TestTuples", inputOperator.outport, o1.inport1);

  lpc.prepareDAG(dag, null, "AutoMetricTest");

  LogicalPlan.OperatorMeta o1meta = dag.getOperatorMeta("o1");
  Assert.assertNotNull("default aggregator injected", o1meta.getMetricAggregatorMeta().getAggregator());

  lpc.prepareDAG(dag, null, "AutoMetricTest");
  StramLocalCluster lc = new StramLocalCluster(dag);
  lc.runAsync();
  latch.await();
  Assert.assertEquals("progress", 1, o1.result.get("progress"));
  lc.shutdown();
}

Example 4

Source File: AutoMetricTest.java From attic-apex-core with Apache License 2.0

6 votes

@Test
public void testMetricsAnnotatedMethod() throws Exception
{
  CountDownLatch latch = new CountDownLatch(1);

  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());

  TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);

  OperatorWithMetricMethod o1 = dag.addOperator("o1", OperatorWithMetricMethod.class);
  MockAggregator aggregator = new MockAggregator(latch);
  dag.setOperatorAttribute(o1, Context.OperatorContext.METRICS_AGGREGATOR, aggregator);

  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  dag.addStream("TestTuples", inputOperator.outport, o1.inport1);

  lpc.prepareDAG(dag, null, "AutoMetricTest");

  StramLocalCluster lc = new StramLocalCluster(dag);
  lc.runAsync();
  latch.await();

  Assert.assertEquals("myMetric", 3, ((Integer)aggregator.result.get("myMetric")).intValue());
  lc.shutdown();
}

Example 5

Source File: TestModuleExpansion.java From attic-apex-core with Apache License 2.0

6 votes

@Test
public void testLoadFromPropertiesFile() throws IOException
{
  Properties props = new Properties();
  String resourcePath = "/testModuleTopology.properties";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    throw new RuntimeException("Could not load " + resourcePath);
  }
  props.load(is);
  LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false))
      .addFromProperties(props, null);

  LogicalPlan dag = new LogicalPlan();
  pb.populateDAG(dag);
  pb.prepareDAG(dag, null, "testApplication");
  dag.validate();
  validateTopLevelOperators(dag);
  validateTopLevelStreams(dag);
  validatePublicMethods(dag);
}

Example 6

Source File: TestModuleExpansion.java From attic-apex-core with Apache License 2.0

6 votes

@Test
public void testLoadFromJson() throws Exception
{
  String resourcePath = "/testModuleTopology.json";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    throw new RuntimeException("Could not load " + resourcePath);
  }
  StringWriter writer = new StringWriter();

  IOUtils.copy(is, writer);
  JSONObject json = new JSONObject(writer.toString());

  Configuration conf = new Configuration(false);
  conf.set(StreamingApplication.APEX_PREFIX + "operator.operator3.prop.myStringProperty", "o3StringFromConf");

  LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
  planConf.prepareDAG(dag, null, "testApplication");
  dag.validate();
  validateTopLevelOperators(dag);
  validateTopLevelStreams(dag);
  validatePublicMethods(dag);
}

Example 7

Source File: StramMiniClusterTest.java From attic-apex-core with Apache License 2.0

5 votes

private LogicalPlan createDAG(LogicalPlanConfiguration lpc) throws Exception
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(LogicalPlan.APPLICATION_PATH, testMeta.toURI().toString());
  lpc.prepareDAG(dag, null, "testApp");
  dag.validate();
  Assert.assertEquals("", Integer.valueOf(128), dag.getValue(DAG.MASTER_MEMORY_MB));
  Assert.assertEquals("", "-Dlog4j.properties=custom_log4j.properties", dag.getValue(DAG.CONTAINER_JVM_OPTIONS));
  return dag;
}

Example 8

Source File: ModuleAppTest.java From attic-apex-core with Apache License 2.0

5 votes

@Test
public void validateTestApplication()
{
  Configuration conf = new Configuration(false);
  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = new LogicalPlan();
  lpc.prepareDAG(dag, new Application(), "TestApp");

  Assert.assertEquals(2, dag.getAllModules().size(), 2);
  Assert.assertEquals(5, dag.getAllOperators().size());
  Assert.assertEquals(4, dag.getAllStreams().size());
  dag.validate();
}

Example 9

Source File: TestModuleExpansion.java From attic-apex-core with Apache License 2.0

5 votes

@Test
public void testModuleExtreme()
{
  StreamingApplication app = new NestedModuleApp();
  Configuration conf = new Configuration(false);
  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = new LogicalPlan();
  lpc.prepareDAG(dag, app, "ModuleApp");

  dag.validate();
  validateTopLevelOperators(dag);
  validateTopLevelStreams(dag);
  validatePublicMethods(dag);
}

Example 10

Source File: TestModuleExpansion.java From attic-apex-core with Apache License 2.0

5 votes

/**
 * Generate a conflict, Add a top level operator with name "m1_O1",
 * and add a module "m1" which will populate operator "O1", causing name conflict with
 * top level operator.
 */
@Test(expected = java.lang.IllegalArgumentException.class)
public void conflictingNamesWithExpandedModule()
{
  Configuration conf = new Configuration(false);
  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = new LogicalPlan();
  DummyInputOperator in = dag.addOperator(componentName("m1", "O1"), new DummyInputOperator());
  Level2ModuleA module = dag.addModule("m1", new Level2ModuleA());
  dag.addStream("s1", in.out, module.mIn);
  lpc.prepareDAG(dag, null, "ModuleApp");
  dag.validate();
}

Example 11

Source File: TestModuleExpansion.java From attic-apex-core with Apache License 2.0

5 votes

/**
 * Module and Operator with same name is not allowed in a DAG, to prevent properties
 * conflict.
 */
@Test(expected = java.lang.IllegalArgumentException.class)
public void conflictingNamesWithOperator1()
{
  Configuration conf = new Configuration(false);
  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = new LogicalPlan();
  DummyInputOperator in = dag.addOperator("M1", new DummyInputOperator());
  Level2ModuleA module = dag.addModule("M1", new Level2ModuleA());
  dag.addStream("s1", in.out, module.mIn);
  lpc.prepareDAG(dag, null, "ModuleApp");
  dag.validate();
}

Example 12

Source File: TestModuleExpansion.java From attic-apex-core with Apache License 2.0

5 votes

/**
 * Module and Operator with same name is not allowed in a DAG, to prevent properties
 * conflict.
 */
@Test(expected = java.lang.IllegalArgumentException.class)
public void conflictingNamesWithOperator2()
{
  Configuration conf = new Configuration(false);
  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = new LogicalPlan();
  Level2ModuleA module = dag.addModule("M1", new Level2ModuleA());
  DummyInputOperator in = dag.addOperator("M1", new DummyInputOperator());
  dag.addStream("s1", in.out, module.mIn);
  lpc.prepareDAG(dag, null, "ModuleApp");
  dag.validate();
}

Example 13

Source File: AutoMetricTest.java From attic-apex-core with Apache License 2.0

4 votes

@Test
public void testInjectionOfDefaultMetricsAggregator() throws Exception
{
  LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());

  TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);

  OperatorWithMetricMethod o1 = dag.addOperator("o1", OperatorWithMetricMethod.class);

  dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  dag.addStream("TestTuples", inputOperator.outport, o1.inport1);

  lpc.prepareDAG(dag, null, "AutoMetricTest");

  LogicalPlan.OperatorMeta o1meta = dag.getOperatorMeta("o1");
  Assert.assertNotNull("default aggregator injected", o1meta.getMetricAggregatorMeta().getAggregator());
}

Example 14

Source File: StreamingAppFactory.java From incubator-samoa with Apache License 2.0

4 votes

public LogicalPlan createApp(LogicalPlanConfiguration planConfig) {
  LogicalPlan dag = new LogicalPlan();
  planConfig.prepareDAG(dag, app, getName());
  return dag;
}