com.datatorrent.api.Context.PortContext Java Examples

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  // RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
  // rand.setMinvalue(0);
  // rand.setMaxvalue(999999);
  // rand.setTuplesBlastIntervalMillis(50);
  // dag.getMeta(rand).getMeta(rand.integer_data).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
  IntegerOperator intInput = dag.addOperator("intInput", new IntegerOperator());
  StreamDuplicater stream = dag.addOperator("stream", new StreamDuplicater());
  dag.getMeta(stream).getMeta(stream.data).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
  dag.addStream("streamdup1", intInput.integer_data, stream.data).setLocality(locality);
  DevNull<Integer> dev1 = dag.addOperator("dev1", new DevNull());
  DevNull<Integer> dev2 = dag.addOperator("dev2", new DevNull());
  dag.addStream("streamdup2", stream.out1, dev1.data).setLocality(locality);
  dag.addStream("streamdup3", stream.out2, dev2.data).setLocality(locality);

}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{

  BlockWriter blockWriter = dag.addOperator("BlockWriter", new BlockWriter());
  Synchronizer synchronizer = dag.addOperator("BlockSynchronizer", new Synchronizer());

  dag.setInputPortAttribute(blockWriter.input, PortContext.PARTITION_PARALLEL, true);
  dag.setInputPortAttribute(blockWriter.blockMetadataInput, PortContext.PARTITION_PARALLEL, true);
  dag.addStream("CompletedBlockmetadata", blockWriter.blockMetadataOutput, synchronizer.blocksMetadataInput);

  HDFSFileMerger merger = new HDFSFileMerger();
  merger = dag.addOperator("FileMerger", merger);
  dag.addStream("MergeTrigger", synchronizer.trigger, merger.input);

  merger.setFilePath(outputDirectoryPath);
  merger.setOverwriteOnConflict(overwriteOnConflict);
  blockWriter.setBlocksDirectory(blocksDirectory);
  merger.setBlocksDirectory(blocksDirectory);

  filesMetadataInput.set(synchronizer.filesMetadataInput);
  blocksMetadataInput.set(blockWriter.blockMetadataInput);
  blockData.set(blockWriter.input);
}
 

public void setup(PortContext context)
{
  setPojoClass(context.getValue(Context.PortContext.TUPLE_CLASS));

  columnFieldSetters = Lists.newArrayList();

  /**
   * Check if the mapping of Generic record fields to POJO is given, else
   * use reflection
   */
  if (getGenericRecordToPOJOFieldsMapping() == null) {
    setFieldInfos(createFieldInfoMap(generateFieldInfoInputs(getPojoClass())));
  } else {
    setFieldInfos(createFieldInfoMap(getGenericRecordToPOJOFieldsMapping()));
  }

  initColumnFieldSetters(getFieldInfos());
  initializeActiveFieldSetters();
}
 

public void setup(PortContext context)
{
  jsonParser = new JSONParser();
  finder = new JsonKeyFinder();
  columnFields = new ArrayList<String>();
  columnFieldSetters = Lists.newArrayList();

  setPojoClass(context.getValue(Context.PortContext.TUPLE_CLASS));

  if (getFieldMappingString() == null) {
    setFieldInfos(createFieldInfoMap(generateFieldInfoInputs(getPojoClass())));
  } else {
    setFieldInfos(createFieldInfoMap(getFieldMappingString()));
  }
  initColumnFieldSetters(getFieldInfos());
  initializeActiveFieldSetters();

  ListIterator<FieldInfo> itr = fieldInfos.listIterator();
  while (itr.hasNext()) {
    columnFields.add(itr.next().getColumnName());
  }
  finder.setMatchKeyList(columnFields);
}
 

@Test
public void testParallelPartitionForSlidingWindow()
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
  GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
  dag.setOperatorAttribute(o1, OperatorContext.SLIDE_BY_WINDOW_COUNT, 2);
  dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<>(2));
  dag.setInputPortAttribute(o2.inport1, PortContext.PARTITION_PARALLEL, true);
  dag.setOperatorAttribute(o1, OperatorContext.APPLICATION_WINDOW_COUNT, 4);

  dag.addStream("o1.outport1", o1.outport1, o2.inport1);
  dag.addStream("o2.outport1", o2.outport1, o3.inport1);
  PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
  Assert.assertEquals("number of containers", 7, plan.getContainers().size());
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{

  RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
  rand.setMaxvalue(3000);
  rand.setTuplesBlast(120);

  RandomMapOutput randMap = dag.addOperator("randMap", new RandomMapOutput());
  randMap.setKey("val");

  RubyOperator ruby = dag.addOperator("ruby", new RubyOperator());
  String setupScript = "def square(val)\n";
  setupScript += "  return val*val\nend\n";
  ruby.addSetupScript(setupScript);
  ruby.setInvoke("square");
  ruby.setPassThru(true);

  ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator());
  dag.getMeta(console).getMeta(console.input).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
  dag.getMeta(ruby).getMeta(ruby.result).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
  dag.addStream("rand_randMap", rand.integer_data, randMap.input).setLocality(Locality.THREAD_LOCAL);
  dag.addStream("randMap_ruby", randMap.map_data, ruby.inBindings).setLocality(locality);
  dag.addStream("ruby_console", ruby.result, console.input).setLocality(locality);
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  JdbcPOJOPollInputOperator poller = dag.addOperator("JdbcPoller", new JdbcPOJOPollInputOperator());

  JdbcStore store = new JdbcStore();
  poller.setStore(store);

  poller.setFieldInfos(addFieldInfos());

  FileLineOutputOperator writer = dag.addOperator("Writer", new FileLineOutputOperator());
  dag.setInputPortAttribute(writer.input, PortContext.PARTITION_PARALLEL, true);
  writer.setRotationWindows(60);

  dag.addStream("dbrecords", poller.outputPort, writer.input);
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  JsonGenerator generator = dag.addOperator("JsonGenerator", JsonGenerator.class);
  JsonParser jsonParser = dag.addOperator("jsonParser", JsonParser.class);

  CsvFormatter formatter = dag.addOperator("formatter", CsvFormatter.class);
  formatter.setSchema(SchemaUtils.jarResourceFileToString(filename));
  dag.setInputPortAttribute(formatter.in, PortContext.TUPLE_CLASS, PojoEvent.class);

  HDFSOutputOperator<String> hdfsOutput = dag.addOperator("HDFSOutputOperator", HDFSOutputOperator.class);
  hdfsOutput.setLineDelimiter("");

  dag.addStream("parserStream", generator.out, jsonParser.in);
  dag.addStream("formatterStream", jsonParser.out, formatter.in);
  dag.addStream("outputStream", formatter.out, hdfsOutput.input);

}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  String filePath = "HDFSOutputOperatorBenchmarkingApp/"
      + System.currentTimeMillis();

  dag.setAttribute(DAG.STREAMING_WINDOW_SIZE_MILLIS, 1000);

  RandomWordGenerator wordGenerator = dag.addOperator("wordGenerator", RandomWordGenerator.class);

  dag.getOperatorMeta("wordGenerator").getMeta(wordGenerator.output)
      .getAttributes().put(PortContext.QUEUE_CAPACITY, 10000);
  dag.getOperatorMeta("wordGenerator").getAttributes()
      .put(OperatorContext.APPLICATION_WINDOW_COUNT, 1);

  FSByteOutputOperator hdfsOutputOperator = dag.addOperator("hdfsOutputOperator", new FSByteOutputOperator());
  hdfsOutputOperator.setFilePath(filePath);
  dag.getOperatorMeta("hdfsOutputOperator").getAttributes()
      .put(OperatorContext.COUNTERS_AGGREGATOR, new BasicCounters.LongAggregator<MutableLong>());

  dag.addStream("Generator2HDFSOutput", wordGenerator.output, hdfsOutputOperator.input);
}
 

@Test
public void testNumberOfUnifiers()
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
  GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
  GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
  dag.addStream("node1.outport1", node1.outport1, node2.inport1);
  dag.setOperatorAttribute(node1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(5));
  dag.setOutputPortAttribute(node1.outport1, PortContext.UNIFIER_LIMIT, 3);
  PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
  List<PTContainer> containers = plan.getContainers();
  int unifierCount = 0;
  int totalOperators = 0;
  for (PTContainer container : containers) {
    List<PTOperator> operators = container.getOperators();
    for (PTOperator operator : operators) {
      totalOperators++;
      if (operator.isUnifier()) {
        unifierCount++;
      }
    }
  }
  Assert.assertEquals("Number of operators", 8, totalOperators);
  Assert.assertEquals("Number of unifiers", 2, unifierCount);
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  EventGenerator eventGenerator = dag.addOperator("eventGenerator", new EventGenerator());
  dag.getMeta(eventGenerator).getMeta(eventGenerator.count).getAttributes()
      .put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);

  DevNull<String> devString = dag.addOperator("devString", new DevNull());
  DevNull<HashMap<String, Double>> devMap = dag.addOperator("devMap", new DevNull());
  DevNull<HashMap<String, Number>> devInt = dag.addOperator("devInt", new DevNull());

  dag.addStream("EventGenString", eventGenerator.string_data, devString.data).setLocality(locality);
  dag.addStream("EventGenMap", eventGenerator.hash_data, devMap.data).setLocality(locality);
  dag.addStream("EventGenInt", eventGenerator.count, devInt.data).setLocality(locality);

}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  WordCountOperator<HashMap<String, Double>> counterString =
      dag.addOperator("counterString", new WordCountOperator<HashMap<String, Double>>());
  dag.getMeta(counterString).getMeta(counterString.input).getAttributes()
      .put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
  EventClassifierNumberToHashDouble eventClassify =
      dag.addOperator("eventClassify", new EventClassifierNumberToHashDouble());
  dag.getMeta(eventClassify).getMeta(eventClassify.data)
      .getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
  IntegerOperator intInput = dag.addOperator("intInput", new IntegerOperator());
  dag.addStream("eventclassifier2", intInput.integer_data, eventClassify.event).setLocality(locality);
  dag.addStream("eventclassifier1", eventClassify.data, counterString.input).setLocality(locality);

}
 

@Test
public void testNumberOfUnifiersWithEvenPartitions()
{
  LogicalPlan dag = new LogicalPlan();
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
  GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
  GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
  dag.addStream("node1.outport1", node1.outport1, node2.inport1);
  dag.setOperatorAttribute(node1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(8));
  dag.setOutputPortAttribute(node1.outport1, PortContext.UNIFIER_LIMIT, 4);
  PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
  List<PTContainer> containers = plan.getContainers();
  int unifierCount = 0;
  int totalOperators = 0;
  for (PTContainer container : containers) {
    List<PTOperator> operators = container.getOperators();
    for (PTOperator operator : operators) {
      totalOperators++;
      if (operator.isUnifier()) {
        unifierCount++;
      }
    }
  }
  Assert.assertEquals("Number of operators", 12, totalOperators);
  Assert.assertEquals("Number of unifiers", 3, unifierCount);
}
 

private Properties propertiesBuilderOutputPort(String attributeName, String root, boolean simpleName, Object val)
{
  String contextClass = simpleName ? null : PortContext.class.getCanonicalName();

  root += "outputport" + LogicalPlanConfiguration.KEY_SEPARATOR + "*" + LogicalPlanConfiguration.KEY_SEPARATOR;

  return propertiesBuilder(attributeName, root, contextClass, val);
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  HiveStore store = new HiveStore();
  store.setDatabaseUrl(conf.get("dt.application.HiveMapInsertBenchmarkingApp.operator.HiveOperator.store.dbUrl"));
  store.setConnectionProperties(conf.get(
      "dt.application.HiveMapInsertBenchmarkingApp.operator.HiveOperator.store.connectionProperties"));
  store.setFilepath(conf.get("dt.application.HiveMapInsertBenchmarkingApp.operator.HiveOperator.store.filepath"));
  try {
    hiveInitializeMapDatabase(store, conf.get(
        "dt.application.HiveMapInsertBenchmarkingApp.operator.HiveOperator.tablename"), ":");
  } catch (SQLException ex) {
    LOG.debug(ex.getMessage());
  }
  dag.setAttribute(DAG.STREAMING_WINDOW_SIZE_MILLIS, 1000);
  RandomEventGenerator eventGenerator = dag.addOperator("EventGenerator", RandomEventGenerator.class);
  RandomMapOutput mapGenerator = dag.addOperator("MapGenerator", RandomMapOutput.class);
  dag.setAttribute(eventGenerator, PortContext.QUEUE_CAPACITY, 10000);
  dag.setAttribute(mapGenerator, PortContext.QUEUE_CAPACITY, 10000);
  HiveOperator hiveInsert = dag.addOperator("HiveOperator", new HiveOperator());
  hiveInsert.setStore(store);
  FSRollingMapTestImpl rollingMapFsWriter = dag.addOperator("RollingFsMapWriter", new FSRollingMapTestImpl());
  rollingMapFsWriter.setFilePath(store.filepath);
  ArrayList<String> hivePartitionColumns = new ArrayList<String>();
  hivePartitionColumns.add("dt");
  hiveInsert.setHivePartitionColumns(hivePartitionColumns);
  dag.addStream("EventGenerator2Map", eventGenerator.integer_data, mapGenerator.input);
  dag.addStream("MapGenerator2HdfsOutput", mapGenerator.map_data, rollingMapFsWriter.input);
  dag.addStream("FsWriter2Hive", rollingMapFsWriter.outputPort, hiveInsert.input);

}
 

@Test
public void testTupleClassAttr() throws Exception
{
  String resourcePath = "/schemaTestTopology.json";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    fail("Could not load " + resourcePath);
  }
  StringWriter writer = new StringWriter();

  IOUtils.copy(is, writer);
  JSONObject json = new JSONObject(writer.toString());

  Configuration conf = new Configuration(false);

  LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
  dag.validate();

  OperatorMeta operator1 = dag.getOperatorMeta("operator1");
  assertEquals("operator1.classname", SchemaTestOperator.class, operator1.getOperator().getClass());

  StreamMeta input1 = dag.getStream("inputStream");
  assertNotNull(input1);
  for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
    Assert.assertEquals("tuple class name required", TestSchema.class, targetPort.getAttributes().get(PortContext.TUPLE_CLASS));
  }
}
 

private Properties propertiesBuilderInputPort(String attributeName, String root, boolean simpleName, Object val)
{
  String contextClass = simpleName ? null : PortContext.class.getCanonicalName();

  root += "inputport" + LogicalPlanConfiguration.KEY_SEPARATOR + "*" + LogicalPlanConfiguration.KEY_SEPARATOR;

  return propertiesBuilder(attributeName, root, contextClass, val);
}
 

private Properties propertiesBuilderPort(String attributeName, String root, boolean simpleName, Object val, boolean addPort)
{
  String contextClass = simpleName ? null : PortContext.class.getCanonicalName();

  if (addPort) {
    root += "port" + LogicalPlanConfiguration.KEY_SEPARATOR + "*" + LogicalPlanConfiguration.KEY_SEPARATOR;
  }

  return propertiesBuilder(attributeName, root, contextClass, val);
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  AvroFileInputOperator avroInputOperator = dag.addOperator("avroInputOperator", getAvroFileInput());
  AvroToPojo avroToPojo = dag.addOperator("AvroToPojo", getAvroToPojo());
  ConsoleOutputOperator consoleOutput = dag.addOperator("GenericRecordOp", new ConsoleOutputOperator());
  dag.getMeta(avroToPojo).getMeta(avroToPojo.output).getAttributes().put(Context.PortContext.TUPLE_CLASS,
      SimpleOrder.class);

  dag.addStream("GenericRecords", avroInputOperator.output, avroToPojo.data).setLocality(Locality.THREAD_LOCAL);
  dag.addStream("POJO", avroToPojo.output, consoleOutput.input).setLocality(Locality.CONTAINER_LOCAL);
}
 

@Override
protected void starting(org.junit.runner.Description description)
{
  String methodName = description.getMethodName();
  String className = description.getClassName();
  this.dir = "target/" + className + "/" + methodName;
  Attribute.AttributeMap attributes = new Attribute.AttributeMap.DefaultAttributeMap();
  attributes.put(Context.DAGContext.APPLICATION_PATH, dir);
  context = mockOperatorContext(1, attributes);
  Attribute.AttributeMap portAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  portAttributes.put(Context.PortContext.TUPLE_CLASS, SimpleOrder.class);
  portContext = new TestPortContext(portAttributes);
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  IntegerOperator intInput = dag.addOperator("intInput", new IntegerOperator());
  StreamMerger stream = dag.addOperator("stream", new StreamMerger());
  dag.getMeta(stream).getMeta(stream.data1).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
  dag.getMeta(stream).getMeta(stream.data2).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
  dag.addStream("streammerge1", intInput.integer_data, stream.data1, stream.data2).setLocality(locality);

  WordCountOperator<Integer> counter = dag.addOperator("counter", new WordCountOperator<Integer>());
  dag.getMeta(counter).getMeta(counter.input).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
  dag.getMeta(stream).getMeta(stream.out).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
  dag.addStream("streammerge2", stream.out, counter.input).setLocality(locality);

}
 

/**
 * Tests both string and non string schema
 *
 * @param dag
 * @param conf
 */
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  // RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator());
  // rand.setMinvalue(0);
  // rand.setMaxvalue(999999);
  // rand.setTuplesBlastIntervalMillis(50);
  // dag.getMeta(rand).getMeta(rand.integer_data).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
  IntegerOperator intInput = dag.addOperator("intInput", new IntegerOperator());
  DevNullCounter oper = dag.addOperator("oper", new DevNullCounter());
  dag.getMeta(oper).getMeta(oper.data).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
  dag.addStream("dev", intInput.integer_data, oper.data).setLocality(locality);

}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  RandomWordInputModule wordGenerator = dag.addOperator("wordGenerator", RandomWordInputModule.class);
  dag.getMeta(wordGenerator).getMeta(wordGenerator.output).getAttributes()
      .put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);

  WordCountOperator<byte[]> counter = dag.addOperator("counter", new WordCountOperator<byte[]>());
  dag.getMeta(counter).getMeta(counter.input).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);

  dag.addStream("Generator2Counter", wordGenerator.output, counter.input).setLocality(getLocality());
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  FixedTuplesInputOperator wordGenerator = dag.addOperator("WordGenerator", FixedTuplesInputOperator.class);
  dag.getMeta(wordGenerator).getMeta(wordGenerator.output).getAttributes()
      .put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);
  wordGenerator.setCount(500000);

  WordCountOperator<byte[]> counter = dag.addOperator("Counter", new WordCountOperator<byte[]>());
  dag.getMeta(counter).getMeta(counter.input).getAttributes().put(PortContext.QUEUE_CAPACITY, QUEUE_CAPACITY);

  dag.addStream("Generator2Counter", wordGenerator.output, counter.input).setLocality(locality);
}
 

@Test
public void testApplication() throws IOException, Exception
{
  LocalMode lma = LocalMode.newInstance();
  new ApplicationFixed().populateDAG(lma.getDAG(), new Configuration(false));

  DAG dag = lma.cloneDAG();
  FixedTuplesInputOperator wordGenerator = (FixedTuplesInputOperator)dag
      .getOperatorMeta("WordGenerator").getOperator();
  Assert.assertEquals("Queue Capacity", ApplicationFixed.QUEUE_CAPACITY, (int)dag.getMeta(wordGenerator)
      .getMeta(wordGenerator.output).getValue(PortContext.QUEUE_CAPACITY));

  LocalMode.Controller lc = lma.getController();
  lc.run(60000);
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  RandomNumberGenerator random = dag.addOperator("randomInt",     RandomNumberGenerator.class);
  TestPartition testPartition  = dag.addOperator("testPartition", TestPartition.class);
  Codec3 codec = new Codec3();
  dag.setInputPortAttribute(testPartition.in, PortContext.STREAM_CODEC, codec);

  //Add locality if needed, e.g.: .setLocality(Locality.CONTAINER_LOCAL);
  dag.addStream("randomData", random.out, testPartition.in);
}
 

/**
 * Populate Yahoo Finance Example Application DAG.
 */
@SuppressWarnings("unchecked")
@Override
public void populateDAG(DAG dag, Configuration conf)
{

  dag.getAttributes().put(DAG.STREAMING_WINDOW_SIZE_MILLIS, streamingWindowSizeMilliSeconds);

  StockTickInput tick = getStockTickInputOperator("StockTickInput", dag);
  SumKeyVal<String, Long> dailyVolume = getDailyVolumeOperator("DailyVolume", dag);
  ConsolidatorKeyVal<String,Double,Long,String,?,?> quoteOperator = getQuoteOperator("Quote", dag);

  RangeKeyVal<String, Double> highlow = getHighLowOperator("HighLow", dag, appWindowCountMinute);
  SumKeyVal<String, Long> minuteVolume = getMinuteVolumeOperator("MinuteVolume", dag, appWindowCountMinute);
  ConsolidatorKeyVal<String,HighLow<Double>,Long,?,?,?> chartOperator = getChartOperator("Chart", dag);

  SimpleMovingAverage<String, Double> priceSMA = getPriceSimpleMovingAverageOperator("PriceSMA", dag, appWindowCountSMA);
  DefaultPartitionCodec<String, Double> codec = new DefaultPartitionCodec<String, Double>();
  dag.setInputPortAttribute(highlow.data, PortContext.STREAM_CODEC, codec);
  dag.setInputPortAttribute(priceSMA.data, PortContext.STREAM_CODEC, codec);
  dag.addStream("price", tick.price, quoteOperator.in1, highlow.data, priceSMA.data);
  dag.addStream("vol", tick.volume, dailyVolume.data, minuteVolume.data);
  dag.addStream("time", tick.time, quoteOperator.in3);
  dag.addStream("daily_vol", dailyVolume.sum, quoteOperator.in2);

  dag.addStream("quote_data", quoteOperator.out, getConsole("quoteConsole", dag, "QUOTE"));

  dag.addStream("high_low", highlow.range, chartOperator.in1);
  dag.addStream("vol_1min", minuteVolume.sum, chartOperator.in2);
  dag.addStream("chart_data", chartOperator.out, getConsole("chartConsole", dag, "CHART"));

  dag.addStream("sma_price", priceSMA.doubleSMA, getConsole("priceSMAConsole", dag, "Price SMA"));

}
 

@Override
public void setup(PortContext context)
{
  cls = context.getValue(Context.PortContext.TUPLE_CLASS);

  try {
    parseSchema();
    initializeColumnMap(getSchema());
  } catch (IOException e) {
    LOG.error("Exception in parsing schema", e);
  }
}
 

@Override
protected void starting(org.junit.runner.Description description)
{
  String methodName = description.getMethodName();
  String className = description.getClassName();
  this.dir = "target/" + className + "/" + methodName;
  Attribute.AttributeMap attributes = new Attribute.AttributeMap.DefaultAttributeMap();
  attributes.put(Context.DAGContext.APPLICATION_PATH, dir);
  context = mockOperatorContext(1, attributes);
  Attribute.AttributeMap portAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  portAttributes.put(Context.PortContext.TUPLE_CLASS, SimpleOrder.class);
  portContext = new TestPortContext(portAttributes);
}
 

public void populateDAG(DAG dag, Configuration conf)
{
  JdbcPOJOPollInputOperator poller = dag.addOperator("JdbcPoller", new JdbcPOJOPollInputOperator());

  JdbcStore store = new JdbcStore();
  poller.setStore(store);

  poller.setFieldInfos(addFieldInfos());

  FileLineOutputOperator writer = dag.addOperator("Writer", new FileLineOutputOperator());
  dag.setInputPortAttribute(writer.input, PortContext.PARTITION_PARALLEL, true);
  writer.setRotationWindows(60);

  dag.addStream("dbrecords", poller.outputPort, writer.input);
}