com.datatorrent.api.LocalMode Java Examples

@Test
public void testCouchBaseAppInput() throws FileNotFoundException, IOException
{
  Configuration conf = new Configuration();
  InputStream is = new FileInputStream("src/site/conf/dt-site-couchbase.xml");
  conf.addResource(is);
  conf.get("dt.application.CouchBaseAppInput.operator.couchbaseInput.store.uriString");
  conf.get("dt.application.CouchBaseAppInput.operator.couchbaseInput.store.blocktime");
  conf.get("dt.application.CouchBaseAppInput.operator.couchbaseInput.store.timeout");
  conf.get("dt.application.CouchBaseAppInput.operator.couchbaseInput.store.bucket");
  conf.get("dt.application.CouchBaseAppInput.operator.couchbaseInput.store.password");
  LocalMode lm = LocalMode.newInstance();

  try {
    lm.prepareDAG(new CouchBaseAppInput(), conf);
    LocalMode.Controller lc = lm.getController();
    lc.run(20000);
  } catch (Exception ex) {
    logger.info(ex.getCause());
  }
  is.close();
}
 

@Test
public void CombinePerKeyExamplesTest() throws Exception
{
  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  conf.set("dt.application.CombinePerKeyExamples.operator.console.silent", "true");
  CombinePerKeyExamples app = new CombinePerKeyExamples();

  lma.prepareDAG(app, conf);

  LocalMode.Controller lc = lma.getController();
  ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
  {
    @Override
    public Boolean call() throws Exception
    {
      return CombinePerKeyExamples.Collector.isDone();
    }
  });
  lc.run(100000);

  Assert.assertTrue(CombinePerKeyExamples.Collector.getResult().get(CombinePerKeyExamples.Collector.getResult().size() - 2).getCorpus().contains("1, 2, 3, 4, 5, 6, 7, 8"));
}
 

@Test
public void testApplication() throws Exception {
  try {
    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(new Application(), getConfig());
    LocalMode.Controller lc = lma.getController();
    lc.runAsync();

    // wait for output files to show up
    while ( ! check(numFiles) ) {
      System.out.println("Sleeping ....");
      Thread.sleep(1000);
    }
  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

@Test
public void testApplication() throws Exception
{
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-SimpleJdbcToHDFSApp.xml"));
    lma.prepareDAG(new JdbcHDFSApp(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.runAsync();

    // wait for output files to roll
    Thread.sleep(5000);

    String[] extensions = {"dat.0","tmp"};
    Collection<File> list = FileUtils.listFiles(new File(FILE_NAME), extensions, false);
    Assert.assertEquals("Records in file", 10, FileUtils.readLines(list.iterator().next()).size());

  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

@Test
public void testCustomParserApp() throws IOException, Exception
{
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(new File("src/test/resources/test-FileToJdbcApp.xml").toURI().toURL());

    lma.prepareDAG(new FileToJdbcCustomParser(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.runAsync(); // test will terminate after results are available

    // wait for records to be added to table
    Thread.sleep(5000);

    Assert.assertEquals("Events in store", 10, getNumOfEventsInStore());
    cleanTable();

  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

@Test
public void validateNegativeMaxLengthTest()
{
  ValidationTestApp validationTestApp = new ValidationTestApp(new File(testMeta.getDir()), -1L,
      new SingleHDFSByteExactlyOnceWriter());

  boolean error = false;

  try {
    LocalMode.runApp(validationTestApp, 1);
  } catch (RuntimeException e) {
    if (e.getCause() instanceof ConstraintViolationException) {
      error = true;
    }
  }

  Assert.assertEquals("Max length validation not thrown with -1 max length", true, error);
}
 

@Test
public void testApplication() throws Exception
{
  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  conf.set("dt.operator.StatefulUniqueCounter.prop.tableName", "Test_Lookup_Cache");
  conf.set("dt.operator.StatefulUniqueCounter.prop.store.dbUrl", "jdbc:hsqldb:mem:test;sql.syntax_mys=true");
  conf.set("dt.operator.StatefulUniqueCounter.prop.store.dbDriver", "org.hsqldb.jdbcDriver");

  lma.prepareDAG(new StatefulApplication(), conf);
  lma.cloneDAG();
  LocalMode.Controller lc = lma.getController();
  lc.setHeartbeatMonitoringEnabled(false);
  lc.runAsync();

  long now = System.currentTimeMillis();
  while (System.currentTimeMillis() - now < 15000) {
    Thread.sleep(1000);
  }

  lc.shutdown();
}
 

@Test
public void AutoCompleteTest() throws Exception
{
  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  conf.set("dt.application.AutoComplete.operator.console.silent", "true");
  lma.prepareDAG(new AutoComplete(), conf);
  LocalMode.Controller lc = lma.getController();

  ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
  {
    @Override
    public Boolean call() throws Exception
    {
      return AutoComplete.Collector.isDone();
    }
  });

  lc.run(200000);

  Assert.assertTrue(AutoComplete.Collector.getResult().containsKey("had"));
  Assert.assertTrue(AutoComplete.Collector.getResult().containsKey("hadoop"));
  Assert.assertEquals(2, AutoComplete.Collector.getResult().get("mapreduce").get(0).getCount());

}
 

@Test
public void TopWikipediaSessionsTest() throws Exception
{
  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  conf.set("dt.application.TopWikipediaSessions.operator.console.silent", "true");
  lma.prepareDAG(new TopWikipediaSessions(), conf);
  LocalMode.Controller lc = lma.getController();

  ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
  {
    @Override
    public Boolean call() throws Exception
    {
      return TopWikipediaSessions.SessionGen.getTupleCount() >= 250;
    }
  });

  lc.run(30000);

  for (int i = 0; i < TopWikipediaSessions.Collector.getResult().size(); i++) {
    Assert.assertTrue(isInOrder(TopWikipediaSessions.Collector.getResult().get(i)));
  }
}
 

@Test
public void testWindowDataManager() throws Exception
{
  // Create DAG for testing.
  LocalMode lma = LocalMode.newInstance();
  DAG dag = lma.getDAG();

  KinesisStringInputOperator inputOperator = dag.addOperator("KinesisInput", new KinesisStringInputOperator()
  {
    @Override
    public void deactivate()
    {
    }

    @Override
    public void teardown()
    {
    }
  });
  testMeta.operator = inputOperator;
  Assert.assertTrue("Default behaviour of WindowDataManager changed",
      (inputOperator.getWindowDataManager() instanceof WindowDataManager.NoopWindowDataManager));
}
 

@Test
public void testCsvParserApp() throws IOException, Exception {
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(new File("src/test/resources/test.xml").toURI().toURL());

    lma.prepareDAG(new FileToJdbcCsvParser(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.runAsync(); // test will terminate after results are available

    // wait for records to be added to table
    Thread.sleep(5000);

    Assert.assertEquals("Events in store", 10, getNumOfEventsInStore());
    cleanTable();

  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

@Test
public void testApplication() throws Exception
{
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    lma.prepareDAG(new JdbcIOApp(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.runAsync();
    // wait for records to be added to table
    Thread.sleep(3000);
    lc.shutdown();
    Assert.assertEquals("Events in store", 10, getNumOfEventsInStore());
  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

@Test(expected = IllegalArgumentException.class)
public void testMissingRecordLength() throws Exception
{
  FixedWidthApplication app = new FixedWidthApplication();
  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  conf.set("dt.operator.S3RecordReaderModuleMock.prop.files", inputDir);
  //Should give IllegalArgumentException since recordLength is not set
  //conf.set("dt.operator.HDFSRecordReaderModule.prop.recordLength", "8");
  conf.set("dt.operator.S3RecordReaderModuleMock.prop.blocksThreshold", "1");
  conf.set("dt.operator.S3RecordReaderModuleMock.prop.blockSize", "3");
  conf.set("dt.operator.S3RecordReaderModuleMock.prop.scanIntervalMillis", "10000");

  lma.prepareDAG(app, conf);
  LocalMode.Controller lc = lma.getController();
  lc.setHeartbeatMonitoringEnabled(true);
  lc.runAsync();
  LOG.debug("Waiting for app to finish");
  Thread.sleep(1000 * 1);
  lc.shutdown();
}
 

@Test
public void testApplication() throws Exception
{
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    lma.prepareDAG(new JdbcToJdbcApp(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.runAsync();

    // wait for records to be added to table    
    Thread.sleep(5000);

    Assert.assertEquals("Events in store", 10, getNumOfEventsInStore());
    cleanTable();

  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

@Test
public void testApplication() throws IOException, Exception {
  try {
    // create file in monitored HDFS directory
    createFile();

    // run app asynchronously; terminate after results are checked
    LocalMode.Controller lc = asyncRun();

    // get messages from Kafka topic and compare with input
    chkOutput();

    lc.shutdown();
  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

public static void main(String[] args) {

    List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));

    args = tmpArgs.toArray(new String[0]);

    ApexTopology apexTopo = ApexSamoaUtils.argsToTopology(args);

    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
//    conf.set("dt.loggers.level", "org.apache.*:DEBUG");

    try {
      lma.prepareDAG(new ApexTask(apexTopo), conf);
      System.out.println("Dag Set in lma: " + lma.getDAG());
      ((LogicalPlan) lma.getDAG()).validate();
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);

    lc.runAsync();
  }
 

@Test
public void testBenchmark() throws FileNotFoundException
{
  Configuration conf = new Configuration();
  InputStream is = new FileInputStream("src/site/conf/dt-site-kafka.xml");
  conf.addResource(is);

  LocalMode lma = LocalMode.newInstance();

  try {
    lma.prepareDAG(new KafkaInputBenchmark(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.run(30000);
  } catch (Exception ex) {
    throw new RuntimeException(ex);
  }
}
 

@Test(expected = IllegalArgumentException.class)
public void testMissingRecordLength() throws Exception
{
  FixedWidthApplication app = new FixedWidthApplication();
  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  conf.set("dt.operator.HDFSRecordReaderModule.prop.files", inputDir);
  //Should give IllegalArgumentException since recordLength is not set
  //conf.set("dt.operator.HDFSRecordReaderModule.prop.recordLength", "8");
  conf.set("dt.operator.HDFSRecordReaderModule.prop.blocksThreshold", "1");
  conf.set("dt.operator.HDFSRecordReaderModule.prop.scanIntervalMillis", "10000");

  lma.prepareDAG(app, conf);
  LocalMode.Controller lc = lma.getController();
  lc.setHeartbeatMonitoringEnabled(true);
  lc.runAsync();
  LOG.debug("Waiting for app to finish");
  Thread.sleep(1000 * 1);
  lc.shutdown();
}
 

@Test
public void testApplication() throws Exception
{
  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  conf.set("dt.operator.Unique.prop.tableName", "Test_Lookup_Cache");
  conf.set("dt.operator.Unique.prop.store.dbUrl", "jdbc:hsqldb:mem:test;sql.syntax_mys=true");
  conf.set("dt.operator.Unique.prop.store.dbDriver", "org.hsqldb.jdbcDriver");

  lma.prepareDAG(new Application(), conf);
  lma.cloneDAG();
  LocalMode.Controller lc = lma.getController();
  lc.setHeartbeatMonitoringEnabled(false);
  lc.runAsync();

  long now = System.currentTimeMillis();
  while (System.currentTimeMillis() - now < 15000) {
    Thread.sleep(1000);
  }
  lc.shutdown();
}
 

@Test
public void testCsvParserApp() throws IOException, Exception
{
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(new File("src/test/resources/test-FileToJdbcApp.xml").toURI().toURL());

    lma.prepareDAG(new FileToJdbcCsvParser(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.runAsync(); // test will terminate after results are available

    // wait for records to be added to table
    Thread.sleep(5000);

    Assert.assertEquals("Events in store", 10, getNumOfEventsInStore());
    cleanTable();

  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

@Test
public void testApplication() throws IOException, Exception {
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    lma.prepareDAG(new Application(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.runAsync();
    while( !check() ) {
      System.out.println("Sleeping...");
      Thread.sleep(1000);
    }
  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

@Test
public void testApplication() throws Exception
{
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/JdbcProperties.xml"));
    lma.prepareDAG(new JdbcPojoOperatorApplication(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
    {
      @Override
      public Boolean call() throws Exception
      {
        return getNumOfRowsinTable(TABLE_POJO_NAME) == 10;
      }
    });
    lc.run(10000);// runs for 10 seconds and quits
    Assert.assertEquals("rows in db", 10, getNumOfRowsinTable(TABLE_POJO_NAME));
  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

@Test
public void testInjectionOfOperatorName() throws Exception
{
  StreamingApplication application = new StreamingApplication()
  {
    @Override
    public void populateDAG(DAG dag, Configuration conf)
    {
      dag.addOperator("input", new MockInputOperator());
    }
  };
  LocalMode lma = LocalMode.newInstance();
  lma.prepareDAG(application, new Configuration());
  LocalMode.Controller lc = lma.getController();
  lc.runAsync();
  latch.await();
  Assert.assertEquals("operator name", "input", operatorName);
  lc.shutdown();
}
 

@Test
public void testEventIncrementerApp() throws FileNotFoundException, IOException
{
  Logger logger = LoggerFactory.getLogger(EventIncrementerAppTest.class);
  LocalMode lm = LocalMode.newInstance();
  Configuration conf = new Configuration();
  InputStream is = new FileInputStream("src/site/conf/dt-site-testbench.xml");
  conf.addResource(is);
  conf.get("dt.application.EventIncrementerApp.operator.hmapOper.seed");
  conf.get("dt.application.EventIncrementerApp.operator.hmapOper.keys");
  conf.get("dt.application.EventIncrementerApp.operator.hmapOper.numKeys");
  try {
    lm.prepareDAG(new EventIncrementerApp(), conf);
    LocalMode.Controller lc = lm.getController();
    lc.run(20000);
  } catch (Exception ex) {
    logger.info(ex.getMessage());
  }
  is.close();
}
 

@Override
public void runEmbedded(boolean async, long duration, Callable<Boolean> exitCondition)
{
  LocalMode lma = LocalMode.newInstance();
  populateDag(lma.getDAG());
  DAG dag = lma.getDAG();
  LocalMode.Controller lc = lma.getController();
  if (lc instanceof StramLocalCluster) {
    ((StramLocalCluster)lc).setExitCondition(exitCondition);
  }
  if (async) {
    lc.runAsync();
  } else {
    if (duration >= 0) {
      lc.run(duration);
    } else {
      lc.run();
    }
  }

}
 

@Test
public void testFilterClassifierApp() throws FileNotFoundException, IOException
{
  Logger logger = LoggerFactory.getLogger(FilteredEventClassifierAppTest.class);
  LocalMode lm = LocalMode.newInstance();
  Configuration conf = new Configuration();
  InputStream is = new FileInputStream("src/site/conf/dt-site-testbench.xml");
  conf.addResource(is);
  conf.get("dt.application.FilteredEventClassifierApp.operator.hmapOper.keys");
  conf.get("dt.application.FilteredEventClassifierApp.operator.hmapOper.numKeys");
  try {
    lm.prepareDAG(new FilteredEventClassifierApp(), conf);
    LocalMode.Controller lc = lm.getController();
    lc.run(20000);
  } catch (Exception ex) {
    logger.info(ex.getMessage());
  }
  is.close();
}
 

@Test
public void DeDupExampleTest() throws Exception
{
  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  conf.set("dt.application.DeDupExample.operator.console.silent", "true");
  DeDupExample app = new DeDupExample();
  lma.prepareDAG(app, conf);
  LocalMode.Controller lc = lma.getController();
  ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
  {
    @Override
    public Boolean call() throws Exception
    {
      return DeDupExample.Collector.isDone();
    }
  });
  lc.run(50000);

  Assert.assertEquals(9, DeDupExample.Collector.getResult().getValue().size());

}
 

@Test
public void testApplication() throws Exception
{
  Configuration conf = new Configuration(false);
  conf.addResource("dt-site-monitoring.xml");
  Server server = new Server(0);
  Servlet servlet = new SamplePubSubWebSocketServlet();
  ServletHolder sh = new ServletHolder(servlet);
  ServletContextHandler contextHandler = new ServletContextHandler(server, "/", ServletContextHandler.SESSIONS);
  contextHandler.addServlet(sh, "/pubsub");
  contextHandler.addServlet(sh, "/*");
  server.start();
  Connector[] connector = server.getConnectors();
  conf.set("dt.attr.GATEWAY_CONNECT_ADDRESS", "localhost:" + connector[0].getLocalPort());

  MRMonitoringApplication application = new MRMonitoringApplication();
  LocalMode lma = LocalMode.newInstance();
  lma.prepareDAG(application, conf);
  LocalMode.Controller lc = lma.getController();
  lc.run(10000);
  server.stop();
}
 

@Test
public void testSomeMethod() throws Exception
{
  Application app = new Application();
  LocalMode lma = LocalMode.newInstance();
  app.populateDAG(lma.getDAG(), new Configuration(false));

  final LocalMode.Controller lc = lma.getController();
  long start = System.currentTimeMillis();
  lc.run();
  long end = System.currentTimeMillis();
  long time = end - start;
  LOG.info("Test used " + time + " ms");
}
 

@Test
public void TrafficRoutesTest() throws Exception
{
  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  conf.set("dt.application.TrafficRoutes.operator.console.silent", "true");
  lma.prepareDAG(new TrafficRoutes(), conf);
  LocalMode.Controller lc = lma.getController();

  ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
  {
    @Override
    public Boolean call() throws Exception
    {
      return TrafficRoutes.InfoGen.getTupleCount() >= 100;
    }
  });

  lc.run(60000);

  Assert.assertTrue(!TrafficRoutes.Collector.getResult().isEmpty());
  for (Map.Entry<KeyValPair<Long, String>, KeyValPair<Double, Boolean>> entry : TrafficRoutes.Collector.getResult().entrySet()) {
    Assert.assertTrue(entry.getValue().getKey() <= 75);
    Assert.assertTrue(entry.getValue().getKey() >= 55);
    Assert.assertTrue(entry.getKey().getValue().equals("SDRoute1") || entry.getKey().getValue().equals("SDRoute2"));
  }
}