Java Code Examples for com.datatorrent.api.Operator#InputPort
The following examples show how to use
com.datatorrent.api.Operator#InputPort .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KuduInputOperatorCommons.java From attic-apex-malhar with Apache License 2.0 | 6 votes |
public static void initCommonConfigsForAllTypesOfTests() throws Exception { KuduClientTestCommons.buildSchemaForUnitTestsTable(); partitioningContext = new Partitioner.PartitioningContext() { @Override public int getParallelPartitionCount() { return numberOfKuduInputOperatorPartitions; } @Override public List<Operator.InputPort<?>> getInputPorts() { return null; } }; }
Example 2
Source File: PlanModifier.java From attic-apex-core with Apache License 2.0 | 6 votes |
public StreamMeta addSinks(String id, Operator.InputPort<?>... sinks) { StreamMeta sm = logicalPlan.getStream(id); if (sm == null) { throw new AssertionError("Stream " + id + " is not found!"); } for (Operator.InputPort<?> sink : sinks) { sm.addSink(sink); if (physicalPlan != null) { for (InputPortMeta ipm : sm.getSinks()) { if (ipm.getPort() == sink) { physicalPlan.connectInput(ipm); } } } } return sm; }
Example 3
Source File: ApexStreamImpl.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
@Override @SuppressWarnings("unchecked") public <O, STREAM extends ApexStream<O>> STREAM addOperator(Operator op, Operator.InputPort<T> inputPort, Operator.OutputPort<O> outputPort, Option... opts) { checkArguments(op, inputPort, outputPort); DagMeta.NodeMeta nm = null; if (lastBrick == null) { nm = graph.addNode(op, null, null, inputPort, opts); } else { nm = graph.addNode(op, lastBrick.nodeMeta, lastBrick.lastOutput, inputPort, opts); } Brick<O> newBrick = new Brick<>(); newBrick.nodeMeta = nm; newBrick.setLastOutput(outputPort); if (lastBrick != null) { newBrick.lastStream = Pair.<Operator.OutputPort, Operator.InputPort>of(lastBrick.lastOutput, inputPort); } if (this.getClass() == ApexStreamImpl.class || this.getClass() == ApexWindowedStreamImpl.class) { return (STREAM)newStream(this.graph, newBrick); } else { try { return (STREAM)this.getClass().getConstructor(ApexStreamImpl.class).newInstance(newStream(this.graph, newBrick)); } catch (Exception e) { throw new RuntimeException("You have to override the default constructor with ApexStreamImpl as default parameter", e); } } }
Example 4
Source File: ApexStreamImpl.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
private void checkArguments(Operator op, Operator.InputPort inputPort, Operator.OutputPort outputPort) { if (op == null) { throw new IllegalArgumentException("Operator can not be null"); } boolean foundInput = inputPort == null; boolean foundOutput = outputPort == null; for (Field f : op.getClass().getFields()) { int modifiers = f.getModifiers(); if (!Modifier.isPublic(modifiers) || !Modifier.isTransient(modifiers)) { continue; } Object obj = null; try { obj = f.get(op); } catch (IllegalAccessException e) { // NonAccessible field is not a valid port object } if (obj == outputPort) { foundOutput = true; } if (obj == inputPort) { foundInput = true; } } if (!foundInput || !foundOutput) { throw new IllegalArgumentException("Input port " + inputPort + " and/or Output port " + outputPort + " is/are not owned by Operator " + op); } }
Example 5
Source File: StreamCodecTest.java From attic-apex-core with Apache License 2.0 | 5 votes |
private void checkPresentStreamCodec(LogicalPlan.OperatorMeta operatorMeta, Operator.InputPort<?> inputPort, Map<Integer, StreamCodec<?>> streamCodecs, String id, PhysicalPlan plan) { StreamCodec<?> streamCodec = operatorMeta.getMeta(inputPort).getStreamCodec(); Assert.assertTrue("stream codec identifier not present" + id, isStrCodecPresent(streamCodec, plan)); Integer streamCodecIdentifier = plan.getStreamCodecIdentifier(streamCodec); checkPresentStreamCodecInfo(streamCodecs, id, streamCodecIdentifier, streamCodec); }
Example 6
Source File: TwitterTopCounterApplication.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
public static void consoleOutput(DAG dag, String operatorName, OutputPort<List<Map<String, Object>>> topCount, String schemaFile, String alias) { if (PubSubHelper.isGatewayConfigured(dag)) { URI uri = PubSubHelper.getURI(dag); AppDataSnapshotServerMap snapshotServer = dag.addOperator("SnapshotServer", new AppDataSnapshotServerMap()); Map<String, String> conversionMap = Maps.newHashMap(); conversionMap.put(alias, WindowedTopCounter.FIELD_TYPE); String snapshotServerJSON = SchemaUtils.jarResourceFileToString(schemaFile); snapshotServer.setSnapshotSchemaJSON(snapshotServerJSON); snapshotServer.setTableFieldToMapField(conversionMap); PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery(); wsQuery.setUri(uri); snapshotServer.setEmbeddableQueryInfoProvider(wsQuery); PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult()); wsResult.setUri(uri); Operator.InputPort<String> queryResultPort = wsResult.input; dag.addStream("MapProvider", topCount, snapshotServer.input); dag.addStream("Result", snapshotServer.queryResult, queryResultPort); } else { ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator()); operator.setStringFormat(operatorName + ": %s"); dag.addStream("MapProvider", topCount, operator.input); } }
Example 7
Source File: RelInfo.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
public RelInfo(String relName, List<Operator.InputPort> inputPorts, Operator operator, Operator.OutputPort outPort, Class clazz) { this.inputPorts = inputPorts; this.operator = operator; this.outPort = outPort; this.clazz = clazz; this.relName = relName; this.outRelDataType = null; }
Example 8
Source File: LogicalPlan.java From attic-apex-core with Apache License 2.0 | 5 votes |
@Override @SuppressWarnings("unchecked") public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T> sink1, Operator.InputPort<? super T> sink2) { @SuppressWarnings("rawtypes") InputPort[] ports = new Operator.InputPort[] {sink1, sink2}; return addStream(id, source, ports); }
Example 9
Source File: LogicalPlan.java From attic-apex-core with Apache License 2.0 | 5 votes |
@Override @SuppressWarnings("unchecked") public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T> sink1) { @SuppressWarnings("rawtypes") InputPort[] ports = new Operator.InputPort[]{sink1}; return addStream(id, source, ports); }
Example 10
Source File: ApexStreamImpl.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
@Override @SuppressWarnings("unchecked") public ApexStream<T> with(DAG.Locality locality) { if (lastBrick.lastStream != null) { for (DagMeta.NodeMeta parent : lastBrick.nodeMeta.getParent()) { Pair<List<Operator.InputPort>, DAG.Locality> p = parent.getNodeStreams().get(lastBrick.lastStream.getLeft()); if (p != null) { p.setValue(locality); } } } return this; }
Example 11
Source File: ApplicationWithQuerySupport.java From attic-apex-malhar with Apache License 2.0 | 4 votes |
/** * Populates the DAG with operators and connecting streams * * @param dag The directed acyclic graph of operators to populate * @param conf The configuration */ @Override public void populateDAG(DAG dag, Configuration conf) { // create operators LineReader lineReader = dag.addOperator("lineReader", new LineReader()); WordReader wordReader = dag.addOperator("wordReader", new WordReader()); WindowWordCount windowWordCount = dag.addOperator("windowWordCount", new WindowWordCount()); FileWordCount fileWordCount = dag.addOperator("fileWordCount", new FileWordCount()); WordCountWriter wcWriter = dag.addOperator("wcWriter", new WordCountWriter()); ConsoleOutputOperator console = dag.addOperator("console", new ConsoleOutputOperator()); console.setStringFormat("wordCount: %s"); // create streams dag.addStream("lines", lineReader.output, wordReader.input); dag.addStream("control", lineReader.control, fileWordCount.control); dag.addStream("words", wordReader.output, windowWordCount.input); dag.addStream("windowWordCounts", windowWordCount.output, fileWordCount.input); dag.addStream("fileWordCounts", fileWordCount.fileOutput, wcWriter.input); if (PubSubHelper.isGatewayConfigured(dag)) { // add query support URI uri = PubSubHelper.getURI(dag); AppDataSnapshotServerMap snapshotServerFile = dag.addOperator("snapshotServerFile", new AppDataSnapshotServerMap()); AppDataSnapshotServerMap snapshotServerGlobal = dag.addOperator("snapshotServerGlobal", new AppDataSnapshotServerMap()); String snapshotServerJSON = SchemaUtils.jarResourceFileToString(SNAPSHOT_SCHEMA); snapshotServerFile.setSnapshotSchemaJSON(snapshotServerJSON); snapshotServerGlobal.setSnapshotSchemaJSON(snapshotServerJSON); PubSubWebSocketAppDataQuery wsQueryFile = new PubSubWebSocketAppDataQuery(); PubSubWebSocketAppDataQuery wsQueryGlobal = new PubSubWebSocketAppDataQuery(); wsQueryFile.setUri(uri); wsQueryGlobal.setUri(uri); snapshotServerFile.setEmbeddableQueryInfoProvider(wsQueryFile); snapshotServerGlobal.setEmbeddableQueryInfoProvider(wsQueryGlobal); PubSubWebSocketAppDataResult wsResultFile = dag.addOperator("wsResultFile", new PubSubWebSocketAppDataResult()); PubSubWebSocketAppDataResult wsResultGlobal = dag.addOperator("wsResultGlobal", new PubSubWebSocketAppDataResult()); wsResultFile.setUri(uri); wsResultGlobal.setUri(uri); Operator.InputPort<String> queryResultFilePort = wsResultFile.input; Operator.InputPort<String> queryResultGlobalPort = wsResultGlobal.input; dag.addStream("WordCountsFile", fileWordCount.outputPerFile, snapshotServerFile.input, console.input); dag.addStream("WordCountsGlobal", fileWordCount.outputGlobal, snapshotServerGlobal.input); dag.addStream("ResultFile", snapshotServerFile.queryResult, queryResultFilePort); dag.addStream("ResultGlobal", snapshotServerGlobal.queryResult, queryResultGlobalPort); } else { //throw new RuntimeException("Error: No GATEWAY_CONNECT_ADDRESS"); dag.addStream("WordCounts", fileWordCount.outputPerFile, console.input); } LOG.info("done with populateDAG, isDebugEnabled = " + LOG.isDebugEnabled()); LOG.info("Returning from populateDAG"); }
Example 12
Source File: RelNodeVisitor.java From attic-apex-malhar with Apache License 2.0 | 4 votes |
/** * This is the main method in this relational node visitor which traverses the relational algebra in reverse direction * and populate the given underlying DAG object. * * @param relNode RelNode which needs to be traversed. * * @return RelInfo representing information of current stage * @throws Exception */ public final RelInfo traverse(RelNode relNode) throws Exception { List<RelInfo> inputStreams = new ArrayList<>(); for (RelNode input : relNode.getInputs()) { inputStreams.add(traverse(input)); } ApexRelNode.RelContext relContext = new ApexRelNode.RelContext(dag, typeFactory, tupleSchemaRegistry); RelInfo currentNodeRelInfo; ApexRelNode apexRelNode = ApexRelNode.relNodeMapping.get(relNode.getClass()); if (apexRelNode == null) { throw new UnsupportedOperationException("RelNode " + relNode.getRelTypeName() + " is not supported."); } currentNodeRelInfo = apexRelNode.visit(relContext, relNode, inputStreams); if (currentNodeRelInfo != null && inputStreams.size() != 0) { for (int i = 0; i < inputStreams.size(); i++) { RelInfo inputStream = inputStreams.get(i); Operator.OutputPort outputPort = inputStream.getOutPort(); Operator.InputPort inputPort = currentNodeRelInfo.getInputPorts().get(i); String streamName = OperatorUtils.getUniqueStreamName(inputStream.getRelName(), currentNodeRelInfo.getRelName()); Class schema; if (inputStream.getOutRelDataType() != null) { schema = TupleSchemaRegistry.getSchemaForRelDataType(tupleSchemaRegistry, streamName, inputStream.getOutRelDataType()); } else if (inputStream.getClazz() != null) { schema = inputStream.getClazz(); } else { throw new RuntimeException("Unexpected condition reached."); } dag.setOutputPortAttribute(outputPort, Context.PortContext.TUPLE_CLASS, schema); dag.setInputPortAttribute(inputPort, Context.PortContext.TUPLE_CLASS, schema); dag.addStream(streamName, outputPort, inputPort); } } if (currentNodeRelInfo.getOutPort() == null) { // End of the pipeline. String schemaJar = tupleSchemaRegistry.generateCommonJar(); String jars = dag.getAttributes().get(Context.DAGContext.LIBRARY_JARS); dag.setAttribute(Context.DAGContext.LIBRARY_JARS, ((jars != null) && (jars.length() != 0)) ? jars + "," + schemaJar : schemaJar); } return currentNodeRelInfo; }
Example 13
Source File: LogicalPlan.java From attic-apex-core with Apache License 2.0 | 4 votes |
@Override public InputPortMeta getMeta(Operator.InputPort<?> port) { return getPortMapping().inPortMap.get(port); }
Example 14
Source File: ApexStreamImpl.java From attic-apex-malhar with Apache License 2.0 | 4 votes |
public void setLastStream(Pair<Operator.OutputPort, Operator.InputPort> lastStream) { this.lastStream = lastStream; }
Example 15
Source File: StreamEndpoint.java From attic-apex-malhar with Apache License 2.0 | 4 votes |
public StreamEndpoint(Operator.InputPort port, Map<String, Class> fieldMapping) { this.inputPort = port; this.fieldMapping = fieldMapping; }
Example 16
Source File: ApexStreamImpl.java From attic-apex-malhar with Apache License 2.0 | 4 votes |
@Override public <O, STREAM extends ApexStream<O>> STREAM endWith(Operator op, Operator.InputPort<T> inputPort, Option... opts) { return (STREAM)addOperator(op, inputPort, null, opts); }
Example 17
Source File: DagMeta.java From attic-apex-malhar with Apache License 2.0 | 4 votes |
public Map<Operator.OutputPort, Pair<List<Operator.InputPort>, DAG.Locality>> getNodeStreams() { return nodeStreams; }
Example 18
Source File: ApexStream.java From attic-apex-malhar with Apache License 2.0 | 2 votes |
/** * Extend the dag by adding one operator<br> * @param op Operator added to the stream * @param inputPort InputPort of the operator that is connected to last exposed OutputPort in the stream * @param outputPort OutputPort of the operator will be connected to next operator * @param <O> type of the output * @return new stream of type O */ <O, STREAM extends ApexStream<O>> STREAM addOperator(Operator op, Operator.InputPort<T> inputPort, Operator.OutputPort<O> outputPort, Option... opts);
Example 19
Source File: ApexStream.java From attic-apex-malhar with Apache License 2.0 | 2 votes |
/** * Extend the dag by adding one end operator<br> * @param op Operator added to the stream * @param inputPort InputPort of the operator that is connected to last exposed OutputPort in the stream * @param <O> type of the output * @return new stream of type O */ <O, STREAM extends ApexStream<O>> STREAM endWith(Operator op, Operator.InputPort<T> inputPort, Option... opts);
Example 20
Source File: Operators.java From attic-apex-core with Apache License 2.0 | votes |
void addInputPort(Operator.InputPort<?> port, Field field, InputPortFieldAnnotation portAnnotation, AppData.QueryPort adqAnnotation);