Java Code Examples for com.datatorrent.api.LocalMode#prepareDAG()

@Test
public void AutoCompleteTest() throws Exception
{
  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  conf.set("dt.application.AutoComplete.operator.console.silent", "true");
  lma.prepareDAG(new AutoComplete(), conf);
  LocalMode.Controller lc = lma.getController();

  ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
  {
    @Override
    public Boolean call() throws Exception
    {
      return AutoComplete.Collector.isDone();
    }
  });

  lc.run(200000);

  Assert.assertTrue(AutoComplete.Collector.getResult().containsKey("had"));
  Assert.assertTrue(AutoComplete.Collector.getResult().containsKey("hadoop"));
  Assert.assertEquals(2, AutoComplete.Collector.getResult().get("mapreduce").get(0).getCount());

}
 

@Test
public void testEventIncrementerApp() throws FileNotFoundException, IOException
{
  Logger logger = LoggerFactory.getLogger(EventIncrementerAppTest.class);
  LocalMode lm = LocalMode.newInstance();
  Configuration conf = new Configuration();
  InputStream is = new FileInputStream("src/site/conf/dt-site-testbench.xml");
  conf.addResource(is);
  conf.get("dt.application.EventIncrementerApp.operator.hmapOper.seed");
  conf.get("dt.application.EventIncrementerApp.operator.hmapOper.keys");
  conf.get("dt.application.EventIncrementerApp.operator.hmapOper.numKeys");
  try {
    lm.prepareDAG(new EventIncrementerApp(), conf);
    LocalMode.Controller lc = lm.getController();
    lc.run(20000);
  } catch (Exception ex) {
    logger.info(ex.getMessage());
  }
  is.close();
}
 

@Test
public void testCouchBaseAppOutput() throws FileNotFoundException, IOException
{
  Configuration conf = new Configuration();
  InputStream is = new FileInputStream("src/site/conf/dt-site-couchbase.xml");
  conf.addResource(is);

  conf.get("dt.application.CouchBaseAppOutput.operator.couchbaseOutput.store.uriString");
  conf.get("dt.application.CouchBaseAppOutput.operator.couchbaseOutput.store.password");
  conf.get("dt.application.CouchBaseAppOutput.operator.couchbaseOutput.store.bucket");
  conf.get("dt.application.couchbaseAppOutput.operator.couchbaseOutput.store.max_tuples");
  conf.get("dt.application.couchbaseAppOutput.operator.couchbaseOutput.store.queueSize");
  conf.get("dt.application.couchbaseAppOutput.operator.couchbaseOutput.store.blocktime");
  conf.get("dt.application.couchbaseAppOutput.operator.couchbaseOutput.store.timeout");
  LocalMode lm = LocalMode.newInstance();

  try {
    lm.prepareDAG(new CouchBaseAppOutput(), conf);
    LocalMode.Controller lc = lm.getController();
    //lc.setHeartbeatMonitoringEnabled(false);
    lc.run(20000);
  } catch (Exception ex) {
    logger.info(ex.getCause());
  }
  is.close();
}
 

@Test
public void testEventClassifierNumberToHashDoubleApp() throws FileNotFoundException, IOException
{
  Logger logger = LoggerFactory.getLogger(EventClassifierNumberToHashDoubleAppTest.class);
  LocalMode lm = LocalMode.newInstance();
  Configuration conf = new Configuration();
  InputStream is = new FileInputStream("src/site/conf/dt-site-testbench.xml");
  conf.addResource(is);
  conf.get("dt.application.EventClassifierNumberToHashDoubleApp.operator.eventClassify.key_keys");
  conf.get("dt.application.EventClassifierNumberToHashDoubleApp.operator.eventClassify.s_start");
  conf.get("dt.application.EventClassifierNumberToHashDoubleApp.operator.eventClassify.s_end");
  try {
    lm.prepareDAG(new EventClassifierNumberToHashDoubleApp(), conf);
    LocalMode.Controller lc = lm.getController();
    lc.run(20000);
  } catch (Exception ex) {
    logger.info(ex.getMessage());
  }
  is.close();
}
 

@Test
public void testCsvParserApp() throws IOException, Exception
{
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(new File("src/test/resources/test-FileToJdbcApp.xml").toURI().toURL());

    lma.prepareDAG(new FileToJdbcCsvParser(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.runAsync(); // test will terminate after results are available

    // wait for records to be added to table
    Thread.sleep(5000);

    Assert.assertEquals("Events in store", 10, getNumOfEventsInStore());
    cleanTable();

  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

private void go(final boolean useUnifier) throws Exception {
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    conf.setBoolean("dt.application.MyFirstApplication.operator.console.prop.saveTuples",
                    true);
    if (useUnifier) {
      conf.setBoolean("dt.application.MyFirstApplication.operator.range.prop.useUnifier",
                      true);
    }
    lma.prepareDAG(new Application(), conf);
    ToConsole console = (ToConsole) lma.getDAG().getOperatorMeta("console").getOperator();
    LocalMode.Controller lc = lma.getController();
    lc.runAsync(); // runs for 10 seconds and quits

    // wait for tuples to show up
    while ( ! check(useUnifier, console) ) {
      System.out.println("Sleeping ....");
      Thread.sleep(500);
    }

  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

@Test
public void testApplication() throws IOException, Exception
{
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    conf.set("dt.application.RecordReaderExample.operator.fileOutput.prop.filePath", outputDir);
    File outputfile = FileUtils.getFile(outputDir, "output.txt_5.0");

    lma.prepareDAG(new Application(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.runAsync();

    // wait for tuples to show up
    while (!outputfile.exists()) {
      Thread.sleep(1000);
    }

    lc.shutdown();
    Assert.assertTrue(FileUtils.contentEquals(
        FileUtils.getFile(
        conf.get("dt.application.RecordReaderExample.operator.recordReader.prop.files")),
        outputfile));

  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

private LocalMode.Controller asyncRun() throws Exception {
  Configuration conf = getConfig();
  LocalMode lma = LocalMode.newInstance();
  lma.prepareDAG(new SqsApplication(), conf);
  LocalMode.Controller lc = lma.getController();
  lc.runAsync();
  return lc;
}
 

@Test
public void testSomeMethod() throws Exception
{
  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  conf.addResource("dt-site-wordcount.xml");
  lma.prepareDAG(new Application(), conf);
  LocalMode.Controller lc = lma.getController();
  long start = System.currentTimeMillis();
  lc.run(300000);
  long end = System.currentTimeMillis();
  long time = end - start;
  LOG.debug("Test used " + time + " ms");
}
 

@Test
@Ignore
public void testApplication() throws IOException, Exception
{
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    lma.prepareDAG(new JdbcToJdbcApp(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.run(50000); // runs for 10 seconds and quits
  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

@Test
public void testApplication() throws IOException, Exception
{

  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    lma.prepareDAG(new Application(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.run(10 * 1000); // runs for 30 seconds and quits
  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

@Test
public void testApplication() throws Exception
{
  EnrichApplication enrichApplication = new EnrichApplication(testMeta);
  enrichApplication.setLoader(testMeta.dbloader);

  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  lma.prepareDAG(enrichApplication, conf);
  LocalMode.Controller lc = lma.getController();
  lc.run(10000);// runs for 10 seconds and quits
}
 

@Test
public void testSomeMethod() throws Exception
{
  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  conf.addResource("dt-site-pi.xml");
  lma.prepareDAG(new Application(), conf);
  LocalMode.Controller lc = lma.getController();
  lc.run(10000);

}
 

@Test
public void testApplication() throws IOException, Exception
{
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    lma.prepareDAG(new Application(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.run(10000); // runs for 10 seconds and quits
  } catch (ConstraintViolationException e) {
    Assert.fail("constraint violations: " + e.getConstraintViolations());
  }
}
 

@Test
public void testApplicationWithPortEndpoint() throws Exception
{
  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    lma.prepareDAG(new KafkaPortApplication(kafka.getBroker(), testTopicData0, testTopicResult), conf);
    LocalMode.Controller lc = lma.getController();
    lc.runAsync();

    kafka.publish(testTopicData0, Arrays.asList("15/02/2016 10:15:00 +0000,1,paint1,11",
        "15/02/2016 10:16:00 +0000,2,paint2,12",
        "15/02/2016 10:17:00 +0000,3,paint3,13", "15/02/2016 10:18:00 +0000,4,paint4,14",
        "15/02/2016 10:19:00 +0000,5,paint5,15", "15/02/2016 10:10:00 +0000,6,abcde6,16"));

    // TODO: Workaround to add \r\n char to test results because of bug in CsvFormatter which adds new line char.
    String[] expectedLines = new String[]{"15/02/2016 10:18:00 +0000,15/02/2016 00:00:00 +0000,OILPAINT4\r\n",
        "15/02/2016 10:19:00 +0000,15/02/2016 00:00:00 +0000,OILPAINT5\r\n"};

    List<String> consume = kafka.consume(testTopicResult, 30000);

    Assert.assertTrue(Arrays.deepEquals(consume.toArray(new String[consume.size()]), expectedLines));

    lc.shutdown();
  } catch (Exception e) {
    Assert.fail("constraint violations: " + e);
  }
}
 

@Test
public void testPopulateDAG() throws Exception
{
  Configuration configuration = new Configuration(false);

  LocalMode lm = LocalMode.newInstance();
  DAG prepareDAG = lm.prepareDAG(new TwitterDumpApplication(), configuration);
  DAG clonedDAG = lm.cloneDAG();

  assertEquals("Serialization", prepareDAG, clonedDAG);
}
 

@Ignore("Skipping because POJOInnerJoinOperator has issues and needs to be replaced with Windowed variant first.")
@Test
public void testApplicationJoinFilter() throws Exception
{
  String sql = "INSERT INTO SALES SELECT STREAM A.ROWTIME, FLOOR(A.ROWTIME TO DAY), " +
      "APEXCONCAT('OILPAINT', SUBSTRING(A.PRODUCT, 6, 7)), B.CATEGORY " +
      "FROM ORDERS AS A JOIN CATEGORY AS B ON A.id = B.id AND A.id > 3" +
      "WHERE A.PRODUCT LIKE 'paint%'";

  try {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    lma.prepareDAG(new KafkaJoinApplication(kafka.getBroker(), testTopicData0, testTopicData1,
        testTopicResult, sql), conf);
    LocalMode.Controller lc = lma.getController();
    lc.runAsync();

    kafka.publish(testTopicData0, Arrays.asList("15/02/2016 10:15:00 +0000,1,paint1,11",
        "15/02/2016 10:16:00 +0000,2,paint2,12",
        "15/02/2016 10:17:00 +0000,3,paint3,13", "15/02/2016 10:18:00 +0000,4,paint4,14",
        "15/02/2016 10:19:00 +0000,5,paint5,15", "15/02/2016 10:10:00 +0000,6,abcde6,16"));

    kafka.publish(testTopicData1, Arrays.asList("1,ABC",
        "2,DEF",
        "3,GHI", "4,JKL",
        "5,MNO", "6,PQR"));

    // TODO: Workaround to add \r\n char to test results because of bug in CsvFormatter which adds new line char.
    String[] expectedLines = new String[]{"15/02/2016 10:18:00 +0000,15/02/2016 00:00:00 +0000,OILPAINT4,JKL\r\n",
        "15/02/2016 10:19:00 +0000,15/02/2016 00:00:00 +0000,OILPAINT5,MNO\r\n"};

    List<String> consume = kafka.consume(testTopicResult, 30000);

    Assert.assertTrue(Arrays.deepEquals(consume.toArray(new String[consume.size()]), expectedLines));

    lc.shutdown();
  } catch (Exception e) {
    Assert.fail("constraint violations: " + e);
  }
}
 

private LocalMode.Controller asyncRun() throws Exception
{
  Configuration conf = getConfig();
  LocalMode lma = LocalMode.newInstance();
  lma.prepareDAG(new KafkaApp(), conf);
  LocalMode.Controller lc = lma.getController();
  lc.runAsync();
  return lc;
}
 

/**
 * This test requires twitter authentication setup and is skipped by default
 * (see {@link TwitterSampleInput}).
 *
 * @throws Exception
 */
@Test
public void testApplication() throws Exception
{
  TwitterTopWordsApplication app = new TwitterTopWordsApplication();
  Configuration conf = new Configuration(false);
  conf.addResource("dt-site-rollingtopwords.xml");
  LocalMode lma = LocalMode.newInstance();
  lma.prepareDAG(app, conf);
  LocalMode.Controller lc = lma.getController();
  lc.run(120000);
}
 

@Test
public void test() throws Exception
{
  LocalMode lma = LocalMode.newInstance();
  Configuration conf = new Configuration(false);
  conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
  conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-PureStyleSQLApplication.xml"));

  conf.set("broker", kafka.getBroker());
  conf.set("topic", testTopicData);
  conf.set("outputFolder", outputFolder);
  conf.set("destFileName", "out.tmp");

  PureStyleSQLApplication app = new PureStyleSQLApplication();

  lma.prepareDAG(app, conf);

  LocalMode.Controller lc = lma.getController();

  lc.runAsync();
  kafka.publish(testTopicData, Arrays.asList(
      "15/02/2016 10:15:00 +0000,1,paint1,11",
      "15/02/2016 10:16:00 +0000,2,paint2,12",
      "15/02/2016 10:17:00 +0000,3,paint3,13",
      "15/02/2016 10:18:00 +0000,4,paint4,14",
      "15/02/2016 10:19:00 +0000,5,paint5,15",
      "15/02/2016 10:10:00 +0000,6,abcde6,16"));

  Assert.assertTrue(waitTillFileIsPopulated(outputFolder, 40000));
  lc.shutdown();

  File file = new File(outputFolder);
  File file1 = new File(outputFolder + file.list()[0]);
  List<String> strings = FileUtils.readLines(file1);

  String[] actualLines = strings.toArray(new String[strings.size()]);

  String[] expectedLines = new String[]{
      "15/02/2016 10:18:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT4",
      "",
      "15/02/2016 10:19:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT5",
      ""};

  Assert.assertEquals(expectedLines.length, actualLines.length);
  for (int i = 0;i < expectedLines.length; i++) {
    Assert.assertEquals(expectedLines[i], actualLines[i]);
  }
}