Java Code Examples for com.datatorrent.api.DAG#setOutputPortAttribute()

The following examples show how to use com.datatorrent.api.DAG#setOutputPortAttribute() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: SimpleTransformApplication.java From attic-apex-malhar with Apache License 2.0

6 votes

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  pojoDataGenerator = dag.addOperator("Input", new POJOGenerator());
  TransformOperator transform = dag.addOperator("Process", new TransformOperator());
  // Set expression map
  Map<String, String> expMap = new HashMap<>();
  expMap.put("name", "{$.firstName}.concat(\" \").concat({$.lastName})");
  expMap.put("age", "(new java.util.Date()).getYear() - {$.dateOfBirth}.getYear()");
  expMap.put("address", "{$.address}.toLowerCase()");
  transform.setExpressionMap(expMap);
  FunctionOperator.MapFunctionOperator<Object, ?> output = dag.addOperator("out",
      new FunctionOperator.MapFunctionOperator<>(outputFn));

  dag.addStream("InputToTransform", pojoDataGenerator.output, transform.input);
  dag.addStream("TransformToOutput", transform.output, output.input);

  dag.setInputPortAttribute(transform.input, Context.PortContext.TUPLE_CLASS, CustomerEvent.class);
  dag.setOutputPortAttribute(transform.output, Context.PortContext.TUPLE_CLASS, CustomerInfo.class);
  setPartitioner(dag,conf,transform);
}

Example 2

Source File: AvroFileToPojoModule.java From attic-apex-malhar with Apache License 2.0

6 votes

@Override
public void populateDAG(DAG dag, Configuration configuration)
{
  AvroFileInputOperator avroFileInputOperator = dag.addOperator("AvroFileInputOperator", this.avroFileInputOperator);
  AvroToPojo avroToPojo = dag.addOperator("AvroGenericObjectToPojo", new AvroToPojo());

  dag.setOutputPortAttribute(avroToPojo.output, Context.PortContext.TUPLE_CLASS, pojoClass);

  dag.addStream("avroFileContainerToPojo", avroFileInputOperator.output, avroToPojo.data)
      .setLocality(DAG.Locality.CONTAINER_LOCAL);

  output.set(avroToPojo.output);
  errorPort.set(avroToPojo.errorPort);

  completedAvroFilesPort.set(avroFileInputOperator.completedFilesPort);
  avroErrorRecordsPort.set(avroFileInputOperator.errorRecordsPort);
}

Example 3

Source File: POJOPartitionJoinOperatorTest.java From attic-apex-malhar with Apache License 2.0

6 votes

@Override
public void populateDAG(DAG dag, Configuration configuration)
{
  TestGenerator gen1 = dag.addOperator("Generator1", new TestGenerator());
  TestGenerator gen2 = dag.addOperator("Generator2", new TestGenerator());

  joinOp = dag.addOperator("Join", new PartitionTestJoinOperator());
  joinOp.setLeftKeyExpression("id");
  joinOp.setRightKeyExpression("id");
  joinOp.setIncludeFieldStr("id,eventTime;id,eventTime");
  joinOp.setExpiryTime(10000L);

  ConsoleOutputOperator console = dag.addOperator("Console", new ConsoleOutputOperator());

  dag.addStream("Gen1ToJoin", gen1.output, joinOp.input1);
  dag.addStream("Gen2ToJoin", gen2.output, joinOp.input2);
  dag.addStream("JoinToConsole", joinOp.outputPort, console.input);
  dag.setInputPortAttribute(joinOp.input1, DAG.InputPortMeta.TUPLE_CLASS,TestEvent.class);
  dag.setInputPortAttribute(joinOp.input2, DAG.InputPortMeta.TUPLE_CLASS,TestEvent.class);
  dag.setOutputPortAttribute(joinOp.outputPort, DAG.InputPortMeta.TUPLE_CLASS,TestEvent.class);
  dag.setAttribute(joinOp, Context.OperatorContext.PARTITIONER,
      new StatelessPartitioner<PartitionTestJoinOperator>(NUM_OF_PARTITIONS));
}

Example 4

Source File: DeduperPartitioningTest.java From attic-apex-malhar with Apache License 2.0

6 votes

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  TestGenerator gen = dag.addOperator("Generator", new TestGenerator());

  dedup = dag.addOperator("Deduper", new TestDeduper());
  dedup.setKeyExpression("id");
  dedup.setTimeExpression("eventTime.getTime()");
  dedup.setBucketSpan(60);
  dedup.setExpireBefore(600);

  ConsoleOutputOperator console = dag.addOperator("Console", new ConsoleOutputOperator());
  dag.addStream("Generator to Dedup", gen.output, dedup.input);
  dag.addStream("Dedup to Console", dedup.unique, console.input);
  dag.setInputPortAttribute(dedup.input, Context.PortContext.TUPLE_CLASS, TestEvent.class);
  dag.setOutputPortAttribute(dedup.unique, Context.PortContext.TUPLE_CLASS, TestEvent.class);
  dag.setAttribute(dedup, Context.OperatorContext.PARTITIONER,
      new StatelessPartitioner<TimeBasedDedupOperator>(NUM_DEDUP_PARTITIONS));
}

Example 5

Source File: RegexParserApplication.java From attic-apex-malhar with Apache License 2.0

5 votes

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  ServerLogGenerator logGenerator = dag.addOperator("logGenerator", ServerLogGenerator.class);
  RegexParser regexParser = dag.addOperator("regexParser", RegexParser.class);
  dag.setOutputPortAttribute(regexParser.out, Context.PortContext.TUPLE_CLASS, ServerLog.class);

  FileOutputOperator regexWriter = dag.addOperator("regexWriter", FileOutputOperator.class);
  FileOutputOperator regexErrorWriter = dag.addOperator("regexErrorWriter", FileOutputOperator.class);

  dag.addStream("regexInput", logGenerator.outputPort, regexParser.in);
  dag.addStream("regexOutput", regexParser.out, regexWriter.input);
  dag.addStream("regexError", regexParser.err, regexErrorWriter.input);
}