Java Code Examples for com.datatorrent.api.Attribute.AttributeMap#DefaultAttributeMap

@Test
public void TestSplunkInputOperator()
{
  SplunkStore store = new SplunkStore();
  store.setHost(HOST);
  store.setPassword(PASSWORD);
  store.setPort(PORT);
  store.setUserName(USER_NAME);

  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(DAG.APPLICATION_ID, APP_ID);
  OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);

  TestInputOperator inputOperator = new TestInputOperator();
  inputOperator.setStore(store);
  inputOperator.setEarliestTime("-1000h");
  inputOperator.setLatestTime("now");
  CollectorTestSink<Object> sink = new CollectorTestSink<Object>();
  inputOperator.outputPort.setSink(sink);

  inputOperator.setup(context);
  inputOperator.beginWindow(0);
  inputOperator.emitTuples();
  inputOperator.endWindow();

  Assert.assertEquals("rows from splunk", 100, sink.collectedTuples.size());
}
 

protected void setupOperator(HBasePOJOPutOperator operator)
{
  configure(operator);

  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_LEAST_ONCE);
  attributeMap.put(OperatorContext.ACTIVATION_WINDOW_ID, -1L);
  attributeMap.put(DAG.APPLICATION_ID, APP_ID);

  OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);

  operator.setup(context);
}
 

@Before
public void setupForTest()
{
  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(DAG.APPLICATION_ID, APP_ID);
  context = mockOperatorContext(OPERATOR_ID, attributeMap);

  Attribute.AttributeMap.DefaultAttributeMap portAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  portAttributes.put(Context.PortContext.TUPLE_CLASS, TestPojo.class);
  tpc = new TestPortContext(portAttributes);
}
 

@Test
public void testHiveInsertMapOperator() throws SQLException, TException
{
  HiveStore hiveStore = createStore(null);
  hiveStore.setFilepath(testdir);
  ArrayList<String> hivePartitionColumns = new ArrayList<String>();
  hivePartitionColumns.add("dt");
  hiveInitializeMapDatabase(createStore(null));
  HiveOperator hiveOperator = new HiveOperator();
  hiveOperator.setStore(hiveStore);
  hiveOperator.setTablename(tablemap);
  hiveOperator.setHivePartitionColumns(hivePartitionColumns);

  FSRollingMapTestImpl fsRolling = new FSRollingMapTestImpl();
  fsRolling.setFilePath(testdir);
  short permission = 511;
  fsRolling.setFilePermission(permission);
  fsRolling.setAlwaysWriteToTmp(false);
  fsRolling.setMaxLength(128);
  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_LEAST_ONCE);
  attributeMap.put(OperatorContext.ACTIVATION_WINDOW_ID, -1L);
  attributeMap.put(DAG.APPLICATION_ID, APP_ID);
  OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);

  fsRolling.setup(context);
  hiveOperator.setup(context);
  HashMap<String, Object> map = new HashMap<String, Object>();
  FilePartitionMapping mapping1 = new FilePartitionMapping();
  FilePartitionMapping mapping2 = new FilePartitionMapping();
  ArrayList<String> partitions1 = new ArrayList<String>();
  partitions1.add("2014-12-10");
  mapping1.setFilename(APP_ID + "/" + OPERATOR_ID + "/" + "2014-12-10" + "/" + "0-transaction.out.part.0");
  mapping1.setPartition(partitions1);
  ArrayList<String> partitions2 = new ArrayList<String>();
  partitions2.add("2014-12-11");
  mapping2.setFilename(APP_ID + "/" + OPERATOR_ID + "/" + "2014-12-11" + "/" + "0-transaction.out.part.0");
  mapping2.setPartition(partitions2);
  for (int wid = 0; wid < NUM_WINDOWS; wid++) {
    fsRolling.beginWindow(wid);
    for (int tupleCounter = 0; tupleCounter < BLAST_SIZE; tupleCounter++) {
      map.put(2014 - 12 - 10 + "", 2014 - 12 - 10);
      fsRolling.input.put(map);
      map.clear();
    }

    if (wid == 7) {
      fsRolling.committed(wid - 1);
      hiveOperator.processTuple(mapping1);
      hiveOperator.processTuple(mapping2);
    }

    fsRolling.endWindow();
  }

  fsRolling.teardown();

  hiveStore.connect();

  client.execute("select * from " + tablemap + " where dt='2014-12-10'");
  List<String> recordsInDatePartition1 = client.fetchAll();

  client.execute("drop table " + tablemap);
  hiveStore.disconnect();

  Assert.assertEquals(13, recordsInDatePartition1.size());
  for (int i = 0; i < recordsInDatePartition1.size(); i++) {
    LOG.debug("records in first date partition are {}", recordsInDatePartition1.get(i));
    /*An array containing partition and data is returned as a string record, hence we need to upcast it to an object first
     and then downcast to a string in order to use in Assert.*/
    Object record = recordsInDatePartition1.get(i);
    Object[] records = (Object[])record;
    Assert.assertEquals("2014-12-10", records[1]);
  }

}
 

static OperatorContext getOperatorContext(final String app_id)
{
  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(DAG.APPLICATION_ID, app_id);
  return mockOperatorContext(OPERATOR_ID, attributeMap);
}
 

@Test
public void testAtleastOnce() throws Exception
{
  try {
    //   HBaseTestHelper.startLocalCluster();
    HBaseTestHelper.clearHBase();
    TestHBasePutOperator thop = new TestHBasePutOperator();

    thop.getStore().setTableName("table1");
    thop.getStore().setZookeeperQuorum("127.0.0.1");
    thop.getStore().setZookeeperClientPort(2181);
    HBaseTuple t1 = new HBaseTuple();
    t1.setColFamily("colfam0");
    t1.setColName("street");
    t1.setRow("row1");
    t1.setColValue("ts");
    HBaseTuple t2 = new HBaseTuple();
    t2.setColFamily("colfam0");
    t2.setColName("city");
    t2.setRow("row2");
    t2.setColValue("tc");
    AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
    attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_LEAST_ONCE);
    thop.setup(mockOperatorContext(0, attributeMap));
    thop.beginWindow(0);
    thop.input.process(t1);
    thop.input.process(t2);
    thop.endWindow();
    HBaseTuple tuple;

    tuple = HBaseTestHelper
        .getHBaseTuple("row1", "colfam0", "street");

    Assert.assertNotNull("Tuple", tuple);
    Assert.assertEquals("Tuple row", tuple.getRow(), "row1");
    Assert.assertEquals("Tuple column family", tuple.getColFamily(), "colfam0");
    Assert.assertEquals("Tuple column name", tuple.getColName(),"street");
    Assert.assertEquals("Tuple column value", tuple.getColValue(),"ts");
  } catch (IOException e) {

    logger.error(e.getMessage());
  }
}
 

@Test
public void testAtmostOnce1() throws Exception
{
  try {
    //   HBaseTestHelper.startLocalCluster();
    HBaseTestHelper.clearHBase();
    TestHBasePutOperator thop = new TestHBasePutOperator();
    thop.getStore().setTableName("table1");
    thop.getStore().setZookeeperQuorum("127.0.0.1");
    thop.getStore().setZookeeperClientPort(2181);
    HBaseTuple t1 = new HBaseTuple();
    t1.setColFamily("colfam0");
    t1.setColName("street");
    t1.setRow("row1");
    t1.setColValue("ts");
    HBaseTuple t2 = new HBaseTuple();
    t2.setColFamily("colfam0");
    t2.setColName("city");
    t2.setRow("row2");
    t2.setColValue("tc");
    AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
    attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_MOST_ONCE);
    thop.setup(mockOperatorContext(0, attributeMap));
    thop.beginWindow(0);
    thop.input.process(t1);
    thop.input.process(t2);
    thop.endWindow();
    HBaseTuple tuple;

    tuple = HBaseTestHelper.getHBaseTuple("row1", "colfam0", "street");

    Assert.assertNotNull("Tuple", tuple);
    Assert.assertEquals("Tuple row", tuple.getRow(), "row1");
    Assert.assertEquals("Tuple column family", tuple.getColFamily(),
        "colfam0");
    Assert.assertEquals("Tuple column name", tuple.getColName(),
        "street");
    Assert.assertEquals("Tuple column value", tuple.getColValue(),
        "ts");
  } catch (IOException e) {

    logger.error(e.getMessage());
  }
}
 

@Test
public void testAtmostOnce2() throws Exception
{
  try {
    //   HBaseTestHelper.startLocalCluster();
    HBaseTestHelper.clearHBase();
    TestHBasePutOperator thop = new TestHBasePutOperator();
    thop.getStore().setTableName("table1");
    thop.getStore().setZookeeperQuorum("127.0.0.1");
    thop.getStore().setZookeeperClientPort(2181);
    HBaseTuple t1 = new HBaseTuple();
    t1.setColFamily("colfam0");
    t1.setColName("street");
    t1.setRow("row1");
    t1.setColValue("ts");
    HBaseTuple t2 = new HBaseTuple();
    t2.setColFamily("colfam0");
    t2.setColName("city");
    t2.setRow("row2");
    t2.setColValue("tc");
    thop.beginWindow(0);
    thop.input.process(t1);
    AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
    attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_MOST_ONCE);
    thop.setup(mockOperatorContext(0, attributeMap));

    thop.input.process(t2);
    thop.endWindow();
    HBaseTuple tuple;
    HBaseTuple tuple2;

    tuple = HBaseTestHelper.getHBaseTuple("row1", "colfam0", "street");
    tuple2 = HBaseTestHelper.getHBaseTuple("row2", "colfam0", "city");
    Assert.assertNull("Tuple", tuple);
    Assert.assertNotNull("Tuple2", tuple2);
    Assert.assertEquals("Tuple row", tuple2.getRow(), "row2");
    Assert.assertEquals("Tuple column family", tuple2.getColFamily(),"colfam0");
    Assert.assertEquals("Tuple column name", tuple2.getColName(),"city");
    Assert.assertEquals("Tuple column value", tuple2.getColValue(),"tc");
  } catch (IOException e) {

    logger.error(e.getMessage());
  }
}
 

@Test
public void testCassandraInputOperator()
{
  String query1 = "SELECT * FROM " + KEYSPACE + "." + "%t;";
  CassandraStore store = new CassandraStore();
  store.setNode(NODE);
  store.setKeyspace(KEYSPACE);

  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(DAG.APPLICATION_ID, APP_ID);
  OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);

  TestInputOperator inputOperator = new TestInputOperator();
  inputOperator.setStore(store);
  inputOperator.setQuery(query1);
  inputOperator.setTablename(TABLE_NAME_INPUT);
  inputOperator.setPrimaryKeyColumn("id");

  List<FieldInfo> fieldInfos = Lists.newArrayList();
  fieldInfos.add(new FieldInfo("id", "id", null));
  fieldInfos.add(new FieldInfo("age", "age", null));
  fieldInfos.add(new FieldInfo("lastname", "lastname", null));
  inputOperator.setFieldInfos(fieldInfos);

  inputOperator.insertEventsInTable(30);
  CollectorTestSink<Object> sink = new CollectorTestSink<>();
  inputOperator.outputPort.setSink(sink);

  Attribute.AttributeMap.DefaultAttributeMap portAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  portAttributes.put(Context.PortContext.TUPLE_CLASS, TestInputPojo.class);
  TestPortContext tpc = new TestPortContext(portAttributes);

  inputOperator.setup(context);
  inputOperator.outputPort.setup(tpc);
  inputOperator.activate(context);

  inputOperator.beginWindow(0);
  inputOperator.emitTuples();
  inputOperator.endWindow();
  Assert.assertEquals("rows from db", 30, sink.collectedTuples.size());
  ArrayList<Integer> listOfIDs = inputOperator.getIds();
  // Rows are not stored in the same order in cassandra table in which they are inserted.
  for (int i = 0; i < 10; i++) {
    TestInputPojo object = (TestInputPojo)sink.collectedTuples.get(i);
    Assert.assertTrue("id set in testpojo", listOfIDs.contains(object.getId()));
    Assert.assertEquals("name set in testpojo", inputOperator.getNames().get(object.getId()), object.getLastname());
    Assert.assertEquals("age set in testpojo", inputOperator.getAge().get(object.getId()).intValue(), object.getAge());
  }

  sink.clear();
  inputOperator.columnDataTypes.clear();

  String query2 = "SELECT * FROM " + KEYSPACE + "." + "%t where token(%p) > %v;";
  inputOperator.setQuery(query2);
  inputOperator.setStartRow(10);
  inputOperator.setup(context);
  inputOperator.outputPort.setup(tpc);
  inputOperator.activate(context);

  inputOperator.beginWindow(1);
  inputOperator.emitTuples();
  inputOperator.endWindow();
  Assert.assertEquals("rows from db", 26, sink.collectedTuples.size());

  sink.clear();
  inputOperator.columnDataTypes.clear();

  String query3 = "SELECT * FROM " + KEYSPACE + "." + "%t where token(%p) > %v LIMIT %l;";
  inputOperator.setQuery(query3);
  inputOperator.setStartRow(1);
  inputOperator.setLimit(10);
  inputOperator.setup(context);
  inputOperator.outputPort.setup(tpc);
  inputOperator.activate(context);

  inputOperator.beginWindow(2);
  inputOperator.emitTuples();
  inputOperator.endWindow();
  Assert.assertEquals("rows from db", 10, sink.collectedTuples.size());
}
 

@Test
public void testMemsqlOutputOperator() throws Exception
{
  cleanDatabase();
  MemsqlStore memsqlStore = createStore(null, true);

  MemsqlPOJOOutputOperator outputOperator = new MemsqlPOJOOutputOperator();
  outputOperator.setStore(memsqlStore);
  outputOperator.setBatchSize(BATCH_SIZE);
  outputOperator.setTablename(FQ_TABLE);
  ArrayList<String> columns = new ArrayList<String>();
  columns.add(DATA_COLUMN1);
  columns.add(DATA_COLUMN2);
  outputOperator.setDataColumns(columns);
  ArrayList<String> expressions = new ArrayList<String>();
  expressions.add("getIntVal()");
  expressions.add("getStringVal()");
  outputOperator.setExpression(expressions);
  AttributeMap.DefaultAttributeMap attributeMap = new AttributeMap.DefaultAttributeMap();
  attributeMap.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_LEAST_ONCE);
  attributeMap.put(OperatorContext.ACTIVATION_WINDOW_ID, -1L);
  attributeMap.put(DAG.APPLICATION_ID, APP_ID);
  OperatorContext context = mockOperatorContext(OPERATOR_ID, attributeMap);

  outputOperator.setup(context);

  for (int wid = 0; wid < NUM_WINDOWS; wid++) {
    outputOperator.beginWindow(wid);
    innerObj.setIntVal(wid + 1);
    outputOperator.input.put(innerObj);

    outputOperator.endWindow();
  }

  outputOperator.teardown();

  memsqlStore.connect();

  int databaseSize;

  Statement statement = memsqlStore.getConnection().createStatement();
  ResultSet resultSet = statement.executeQuery("select count(*) from " + FQ_TABLE);
  resultSet.next();
  databaseSize = resultSet.getInt(1);

  memsqlStore.disconnect();

  Assert.assertEquals("Numer of tuples in database", DATABASE_SIZE, databaseSize);
}