Java Code Examples for com.datatorrent.api.Operator#InputPort

The following examples show how to use com.datatorrent.api.Operator#InputPort . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KuduInputOperatorCommons.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
public static void initCommonConfigsForAllTypesOfTests() throws Exception
{
  KuduClientTestCommons.buildSchemaForUnitTestsTable();
  partitioningContext = new Partitioner.PartitioningContext()
  {
    @Override
    public int getParallelPartitionCount()
    {
      return numberOfKuduInputOperatorPartitions;
    }

    @Override
    public List<Operator.InputPort<?>> getInputPorts()
    {
      return null;
    }
  };
}
 
Example 2
Source File: PlanModifier.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
public StreamMeta addSinks(String id, Operator.InputPort<?>... sinks)
{
  StreamMeta sm = logicalPlan.getStream(id);
  if (sm == null) {
    throw new AssertionError("Stream " + id + " is not found!");
  }
  for (Operator.InputPort<?> sink : sinks) {
    sm.addSink(sink);
    if (physicalPlan != null) {
      for (InputPortMeta ipm : sm.getSinks()) {
        if (ipm.getPort() == sink) {
          physicalPlan.connectInput(ipm);
        }
      }
    }
  }
  return sm;
}
 
Example 3
Source File: ApexStreamImpl.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public  <O, STREAM extends ApexStream<O>> STREAM addOperator(Operator op, Operator.InputPort<T> inputPort, Operator.OutputPort<O> outputPort, Option... opts)
{

  checkArguments(op, inputPort, outputPort);

  DagMeta.NodeMeta nm = null;

  if (lastBrick == null) {
    nm = graph.addNode(op, null, null, inputPort, opts);
  } else {
    nm = graph.addNode(op, lastBrick.nodeMeta, lastBrick.lastOutput, inputPort, opts);
  }

  Brick<O> newBrick = new Brick<>();
  newBrick.nodeMeta = nm;
  newBrick.setLastOutput(outputPort);
  if (lastBrick != null) {
    newBrick.lastStream = Pair.<Operator.OutputPort, Operator.InputPort>of(lastBrick.lastOutput, inputPort);
  }

  if (this.getClass() == ApexStreamImpl.class || this.getClass() == ApexWindowedStreamImpl.class) {
    return (STREAM)newStream(this.graph, newBrick);
  } else {
    try {
      return (STREAM)this.getClass().getConstructor(ApexStreamImpl.class).newInstance(newStream(this.graph, newBrick));
    } catch (Exception e) {
      throw new RuntimeException("You have to override the default constructor with ApexStreamImpl as default parameter", e);
    }
  }

}
 
Example 4
Source File: ApexStreamImpl.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
private void checkArguments(Operator op, Operator.InputPort inputPort, Operator.OutputPort outputPort)
{
  if (op == null) {
    throw new IllegalArgumentException("Operator can not be null");
  }

  boolean foundInput = inputPort == null;
  boolean foundOutput = outputPort == null;
  for (Field f : op.getClass().getFields()) {
    int modifiers = f.getModifiers();
    if (!Modifier.isPublic(modifiers) || !Modifier.isTransient(modifiers)) {
      continue;
    }
    Object obj = null;
    try {
      obj = f.get(op);
    } catch (IllegalAccessException e) {
      // NonAccessible field is not a valid port object
    }
    if (obj == outputPort) {
      foundOutput = true;
    }
    if (obj == inputPort) {
      foundInput = true;
    }
  }
  if (!foundInput || !foundOutput) {
    throw new IllegalArgumentException("Input port " + inputPort + " and/or Output port " + outputPort + " is/are not owned by Operator " + op);
  }

}
 
Example 5
Source File: StreamCodecTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
private void checkPresentStreamCodec(LogicalPlan.OperatorMeta operatorMeta, Operator.InputPort<?> inputPort,
    Map<Integer, StreamCodec<?>> streamCodecs,
    String id, PhysicalPlan plan)
{
  StreamCodec<?> streamCodec = operatorMeta.getMeta(inputPort).getStreamCodec();
  Assert.assertTrue("stream codec identifier not present" + id, isStrCodecPresent(streamCodec, plan));
  Integer streamCodecIdentifier = plan.getStreamCodecIdentifier(streamCodec);
  checkPresentStreamCodecInfo(streamCodecs, id, streamCodecIdentifier, streamCodec);
}
 
Example 6
Source File: TwitterTopCounterApplication.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
public static void consoleOutput(DAG dag, String operatorName, OutputPort<List<Map<String, Object>>> topCount, String schemaFile, String alias)
{
  if (PubSubHelper.isGatewayConfigured(dag)) {
    URI uri = PubSubHelper.getURI(dag);

    AppDataSnapshotServerMap snapshotServer = dag.addOperator("SnapshotServer", new AppDataSnapshotServerMap());

    Map<String, String> conversionMap = Maps.newHashMap();
    conversionMap.put(alias, WindowedTopCounter.FIELD_TYPE);
    String snapshotServerJSON = SchemaUtils.jarResourceFileToString(schemaFile);

    snapshotServer.setSnapshotSchemaJSON(snapshotServerJSON);
    snapshotServer.setTableFieldToMapField(conversionMap);

    PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery();
    wsQuery.setUri(uri);
    snapshotServer.setEmbeddableQueryInfoProvider(wsQuery);

    PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());
    wsResult.setUri(uri);
    Operator.InputPort<String> queryResultPort = wsResult.input;

    dag.addStream("MapProvider", topCount, snapshotServer.input);
    dag.addStream("Result", snapshotServer.queryResult, queryResultPort);
  } else {
    ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator());
    operator.setStringFormat(operatorName + ": %s");

    dag.addStream("MapProvider", topCount, operator.input);
  }
}
 
Example 7
Source File: RelInfo.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
public RelInfo(String relName, List<Operator.InputPort> inputPorts, Operator operator, Operator.OutputPort outPort, Class clazz)
{
  this.inputPorts = inputPorts;
  this.operator = operator;
  this.outPort = outPort;
  this.clazz = clazz;
  this.relName = relName;
  this.outRelDataType = null;
}
 
Example 8
Source File: LogicalPlan.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T> sink1, Operator.InputPort<? super T> sink2)
{
  @SuppressWarnings("rawtypes")
  InputPort[] ports = new Operator.InputPort[] {sink1, sink2};
  return addStream(id, source, ports);
}
 
Example 9
Source File: LogicalPlan.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T> sink1)
{
  @SuppressWarnings("rawtypes")
  InputPort[] ports = new Operator.InputPort[]{sink1};
  return addStream(id, source, ports);
}
 
Example 10
Source File: ApexStreamImpl.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public ApexStream<T> with(DAG.Locality locality)
{
  if (lastBrick.lastStream != null) {
    for (DagMeta.NodeMeta parent : lastBrick.nodeMeta.getParent()) {
      Pair<List<Operator.InputPort>, DAG.Locality> p = parent.getNodeStreams().get(lastBrick.lastStream.getLeft());
      if (p != null) {
        p.setValue(locality);
      }
    }
  }
  return this;
}
 
Example 11
Source File: ApplicationWithQuerySupport.java    From attic-apex-malhar with Apache License 2.0 4 votes vote down vote up
/**
 * Populates the DAG with operators and connecting streams
 *
 * @param dag The directed acyclic graph of operators to populate
 * @param conf The configuration
 */
@Override
public void populateDAG(DAG dag, Configuration conf)
{
  // create operators
  LineReader lineReader            = dag.addOperator("lineReader", new LineReader());
  WordReader wordReader            = dag.addOperator("wordReader", new WordReader());
  WindowWordCount windowWordCount  = dag.addOperator("windowWordCount", new WindowWordCount());
  FileWordCount fileWordCount      = dag.addOperator("fileWordCount", new FileWordCount());
  WordCountWriter wcWriter         = dag.addOperator("wcWriter", new WordCountWriter());
  ConsoleOutputOperator console    = dag.addOperator("console", new ConsoleOutputOperator());
  console.setStringFormat("wordCount: %s");

  // create streams

  dag.addStream("lines",   lineReader.output,  wordReader.input);
  dag.addStream("control", lineReader.control, fileWordCount.control);
  dag.addStream("words",   wordReader.output,  windowWordCount.input);
  dag.addStream("windowWordCounts", windowWordCount.output, fileWordCount.input);
  dag.addStream("fileWordCounts", fileWordCount.fileOutput, wcWriter.input);

  if (PubSubHelper.isGatewayConfigured(dag)) {        // add query support
    URI uri = PubSubHelper.getURI(dag);

    AppDataSnapshotServerMap snapshotServerFile
        = dag.addOperator("snapshotServerFile", new AppDataSnapshotServerMap());
    AppDataSnapshotServerMap snapshotServerGlobal
        = dag.addOperator("snapshotServerGlobal", new AppDataSnapshotServerMap());

    String snapshotServerJSON = SchemaUtils.jarResourceFileToString(SNAPSHOT_SCHEMA);
    snapshotServerFile.setSnapshotSchemaJSON(snapshotServerJSON);
    snapshotServerGlobal.setSnapshotSchemaJSON(snapshotServerJSON);

    PubSubWebSocketAppDataQuery wsQueryFile = new PubSubWebSocketAppDataQuery();
    PubSubWebSocketAppDataQuery wsQueryGlobal = new PubSubWebSocketAppDataQuery();
    wsQueryFile.setUri(uri);
    wsQueryGlobal.setUri(uri);

    snapshotServerFile.setEmbeddableQueryInfoProvider(wsQueryFile);
    snapshotServerGlobal.setEmbeddableQueryInfoProvider(wsQueryGlobal);

    PubSubWebSocketAppDataResult wsResultFile
        = dag.addOperator("wsResultFile", new PubSubWebSocketAppDataResult());
    PubSubWebSocketAppDataResult wsResultGlobal
        = dag.addOperator("wsResultGlobal", new PubSubWebSocketAppDataResult());
    wsResultFile.setUri(uri);
    wsResultGlobal.setUri(uri);

    Operator.InputPort<String> queryResultFilePort = wsResultFile.input;
    Operator.InputPort<String> queryResultGlobalPort = wsResultGlobal.input;

    dag.addStream("WordCountsFile", fileWordCount.outputPerFile, snapshotServerFile.input, console.input);
    dag.addStream("WordCountsGlobal", fileWordCount.outputGlobal, snapshotServerGlobal.input);

    dag.addStream("ResultFile", snapshotServerFile.queryResult, queryResultFilePort);
    dag.addStream("ResultGlobal", snapshotServerGlobal.queryResult, queryResultGlobalPort);
  } else {
    //throw new RuntimeException("Error: No GATEWAY_CONNECT_ADDRESS");
    dag.addStream("WordCounts", fileWordCount.outputPerFile, console.input);
  }

  LOG.info("done with populateDAG, isDebugEnabled = " + LOG.isDebugEnabled());
  LOG.info("Returning from populateDAG");
}
 
Example 12
Source File: RelNodeVisitor.java    From attic-apex-malhar with Apache License 2.0 4 votes vote down vote up
/**
 * This is the main method in this relational node visitor which traverses the relational algebra in reverse direction
 * and populate the given underlying DAG object.
 *
 * @param relNode RelNode which needs to be traversed.
 *
 * @return RelInfo representing information of current stage
 * @throws Exception
 */
public final RelInfo traverse(RelNode relNode) throws Exception
{
  List<RelInfo> inputStreams = new ArrayList<>();
  for (RelNode input : relNode.getInputs()) {
    inputStreams.add(traverse(input));
  }

  ApexRelNode.RelContext relContext = new ApexRelNode.RelContext(dag, typeFactory, tupleSchemaRegistry);

  RelInfo currentNodeRelInfo;
  ApexRelNode apexRelNode = ApexRelNode.relNodeMapping.get(relNode.getClass());
  if (apexRelNode == null) {
    throw new UnsupportedOperationException("RelNode " + relNode.getRelTypeName() + " is not supported.");
  }
  currentNodeRelInfo = apexRelNode.visit(relContext, relNode, inputStreams);

  if (currentNodeRelInfo != null && inputStreams.size() != 0) {
    for (int i = 0; i < inputStreams.size(); i++) {
      RelInfo inputStream = inputStreams.get(i);
      Operator.OutputPort outputPort = inputStream.getOutPort();
      Operator.InputPort inputPort = currentNodeRelInfo.getInputPorts().get(i);

      String streamName = OperatorUtils.getUniqueStreamName(inputStream.getRelName(),
          currentNodeRelInfo.getRelName());
      Class schema;
      if (inputStream.getOutRelDataType() != null) {
        schema = TupleSchemaRegistry.getSchemaForRelDataType(tupleSchemaRegistry, streamName,
            inputStream.getOutRelDataType());
      } else if (inputStream.getClazz() != null) {
        schema = inputStream.getClazz();
      } else {
        throw new RuntimeException("Unexpected condition reached.");
      }
      dag.setOutputPortAttribute(outputPort, Context.PortContext.TUPLE_CLASS, schema);
      dag.setInputPortAttribute(inputPort, Context.PortContext.TUPLE_CLASS, schema);
      dag.addStream(streamName, outputPort, inputPort);
    }
  }

  if (currentNodeRelInfo.getOutPort() == null) {
    // End of the pipeline.
    String schemaJar = tupleSchemaRegistry.generateCommonJar();

    String jars = dag.getAttributes().get(Context.DAGContext.LIBRARY_JARS);
    dag.setAttribute(Context.DAGContext.LIBRARY_JARS,
        ((jars != null) && (jars.length() != 0)) ? jars + "," + schemaJar : schemaJar);
  }

  return currentNodeRelInfo;
}
 
Example 13
Source File: LogicalPlan.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up
@Override
public InputPortMeta getMeta(Operator.InputPort<?> port)
{
  return getPortMapping().inPortMap.get(port);
}
 
Example 14
Source File: ApexStreamImpl.java    From attic-apex-malhar with Apache License 2.0 4 votes vote down vote up
public void setLastStream(Pair<Operator.OutputPort, Operator.InputPort> lastStream)
{
  this.lastStream = lastStream;
}
 
Example 15
Source File: StreamEndpoint.java    From attic-apex-malhar with Apache License 2.0 4 votes vote down vote up
public StreamEndpoint(Operator.InputPort port, Map<String, Class> fieldMapping)
{
  this.inputPort = port;
  this.fieldMapping = fieldMapping;
}
 
Example 16
Source File: ApexStreamImpl.java    From attic-apex-malhar with Apache License 2.0 4 votes vote down vote up
@Override
public <O, STREAM extends ApexStream<O>> STREAM endWith(Operator op, Operator.InputPort<T> inputPort, Option... opts)
{
  return (STREAM)addOperator(op, inputPort, null, opts);
}
 
Example 17
Source File: DagMeta.java    From attic-apex-malhar with Apache License 2.0 4 votes vote down vote up
public Map<Operator.OutputPort, Pair<List<Operator.InputPort>, DAG.Locality>> getNodeStreams()
{
  return nodeStreams;
}
 
Example 18
Source File: ApexStream.java    From attic-apex-malhar with Apache License 2.0 2 votes vote down vote up
/**
 * Extend the dag by adding one operator<br>
 * @param op Operator added to the stream
 * @param inputPort InputPort of the operator that is connected to last exposed OutputPort in the stream
 * @param outputPort OutputPort of the operator will be connected to next operator
 * @param <O> type of the output
 * @return new stream of type O
 */
<O, STREAM extends ApexStream<O>> STREAM addOperator(Operator op, Operator.InputPort<T> inputPort,  Operator.OutputPort<O> outputPort, Option... opts);
 
Example 19
Source File: ApexStream.java    From attic-apex-malhar with Apache License 2.0 2 votes vote down vote up
/**
 * Extend the dag by adding one end operator<br>
 * @param op Operator added to the stream
 * @param inputPort InputPort of the operator that is connected to last exposed OutputPort in the stream
 * @param <O> type of the output
 * @return new stream of type O
 */
<O, STREAM extends ApexStream<O>> STREAM endWith(Operator op, Operator.InputPort<T> inputPort, Option... opts);
 
Example 20
Source File: Operators.java    From attic-apex-core with Apache License 2.0 votes vote down vote up
void addInputPort(Operator.InputPort<?> port, Field field, InputPortFieldAnnotation portAnnotation, AppData.QueryPort adqAnnotation);