com.datatorrent.api.Operator Java Examples
The following examples show how to use
com.datatorrent.api.Operator.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StramWebServices.java From attic-apex-core with Apache License 2.0 | 6 votes |
@GET @Path(PATH_OPERATOR_CLASSES + "/{className}") @Produces(MediaType.APPLICATION_JSON) @SuppressWarnings("unchecked") public JSONObject describeOperator(@PathParam("className") String className) { init(); if (className == null) { throw new UnsupportedOperationException(); } try { Class<?> clazz = Class.forName(className); if (Operator.class.isAssignableFrom(clazz)) { return operatorDiscoverer.describeOperator(className); } else { throw new NotFoundException(); } } catch (Exception ex) { throw new NotFoundException(); } }
Example #2
Source File: LogicalPlan.java From attic-apex-core with Apache License 2.0 | 6 votes |
@Override public <T extends Operator> T addOperator(@Nonnull String name, T operator) { checkArgument(!isNullOrEmpty(name), "operator name is null or empty"); if (operators.containsKey(name)) { if (operators.get(name).operator == operator) { return operator; } throw new IllegalArgumentException("duplicate operator name: " + operators.get(name)); } // Avoid name conflict with module. if (modules.containsKey(name)) { throw new IllegalArgumentException("duplicate operator name: " + operators.get(name)); } OperatorMeta decl = new OperatorMeta(name, operator); rootOperators.add(decl); // will be removed when a sink is added to an input port for this operator leafOperators.add(decl); // will be removed when a sink is added to an output port for this operator operators.put(name, decl); return operator; }
Example #3
Source File: TypeGraph.java From attic-apex-core with Apache License 2.0 | 6 votes |
public List<CompactFieldNode> getAllOutputPorts(String clazzName) { TypeGraphVertex tgv = typeGraph.get(clazzName); List<CompactFieldNode> ports = new ArrayList<>(); TypeGraphVertex portVertex = typeGraph.get(Operator.OutputPort.class.getName()); getAllPortsWithAncestor(portVertex, tgv, ports); Collections.sort(ports, new Comparator<CompactFieldNode>() { @Override public int compare(CompactFieldNode a, CompactFieldNode b) { return a.getName().compareTo(b.getName()); } }); return ports; }
Example #4
Source File: StramToNodeGetPropertyRequest.java From attic-apex-core with Apache License 2.0 | 6 votes |
@Override public StatsListener.OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException { BeanMap beanMap = new BeanMap(operator); Map<String, Object> propertyValue = new HashMap<>(); if (propertyName != null) { if (beanMap.containsKey(propertyName)) { propertyValue.put(propertyName, beanMap.get(propertyName)); } } else { Iterator entryIterator = beanMap.entryIterator(); while (entryIterator.hasNext()) { Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next(); propertyValue.put(entry.getKey(), entry.getValue()); } } logger.debug("Getting property {} on operator {}", propertyValue, operator); OperatorResponse response = new OperatorResponse(requestId, propertyValue); return response; }
Example #5
Source File: LogicalPlan.java From attic-apex-core with Apache License 2.0 | 6 votes |
public void removeOperator(Operator operator) { OperatorMeta om = getMeta(operator); if (om == null) { return; } Map<InputPortMeta, StreamMeta> inputStreams = om.getInputStreams(); for (Map.Entry<InputPortMeta, StreamMeta> e : inputStreams.entrySet()) { StreamMeta stream = e.getValue(); if (e.getKey().getOperatorMeta() == om) { stream.sinks.remove(e.getKey()); } // If persistStream was enabled for stream, reset stream when sink removed stream.resetStreamPersistanceOnSinkRemoval(e.getKey()); } this.operators.remove(om.getName()); rootOperators.remove(om); leafOperators.remove(om); }
Example #6
Source File: PlanModifier.java From attic-apex-core with Apache License 2.0 | 6 votes |
public StreamMeta addSinks(String id, Operator.InputPort<?>... sinks) { StreamMeta sm = logicalPlan.getStream(id); if (sm == null) { throw new AssertionError("Stream " + id + " is not found!"); } for (Operator.InputPort<?> sink : sinks) { sm.addSink(sink); if (physicalPlan != null) { for (InputPortMeta ipm : sm.getSinks()) { if (ipm.getPort() == sink) { physicalPlan.connectInput(ipm); } } } } return sm; }
Example #7
Source File: StatelessPartitioner.java From attic-apex-core with Apache License 2.0 | 6 votes |
/** * Adjust the partitions of an input operator (operator with no connected input stream). * * @param <T> The operator type * @param partitions * @return The new operators. */ public static <T extends Operator> Collection<Partition<T>> repartitionInputOperator(Collection<Partition<T>> partitions) { List<Partition<T>> newPartitions = new ArrayList<>(); List<Partition<T>> lowLoadPartitions = new ArrayList<>(); for (Partition<T> p: partitions) { int load = p.getLoad(); if (load < 0) { if (!lowLoadPartitions.isEmpty()) { newPartitions.add(lowLoadPartitions.remove(0)); } else { lowLoadPartitions.add(p); } } else if (load > 0) { newPartitions.add(new DefaultPartition<>(p.getPartitionedInstance())); newPartitions.add(new DefaultPartition<>(p.getPartitionedInstance())); } else { newPartitions.add(p); } } newPartitions.addAll(lowLoadPartitions); return newPartitions; }
Example #8
Source File: LogicalPlan.java From attic-apex-core with Apache License 2.0 | 6 votes |
public Operator.Unifier<?> getUnifier() { for (Map.Entry<OutputPort<?>, OutputPortMeta> e : operatorMeta.getPortMapping().outPortMap.entrySet()) { if (e.getValue() == this) { Unifier<?> unifier = e.getKey().getUnifier(); if (unifier == null) { break; } LOG.debug("User supplied unifier is {}", unifier); return unifier; } } LOG.debug("Using default unifier for {}", this); return new DefaultUnifier(); }
Example #9
Source File: LogicalPlan.java From attic-apex-core with Apache License 2.0 | 6 votes |
@Override public StreamMeta setSource(Operator.OutputPort<?> port) { if (port instanceof ProxyOutputPort) { proxySource = port; return this; } OutputPortMeta portMeta = assertGetPortMeta(port); OperatorMeta om = portMeta.getOperatorMeta(); if (om.outputStreams.containsKey(portMeta)) { String msg = String.format("Operator %s already connected to %s", om.name, om.outputStreams.get(portMeta).id); throw new IllegalArgumentException(msg); } this.source = portMeta; om.outputStreams.put(portMeta, this); return this; }
Example #10
Source File: PhysicalPlan.java From attic-apex-core with Apache License 2.0 | 6 votes |
private void initCheckpoint(PTOperator oper, Operator oo, Checkpoint checkpoint) { try { LOG.debug("Writing activation checkpoint {} {} {}", checkpoint, oper, oo); long windowId = oper.isOperatorStateLess() ? Stateless.WINDOW_ID : checkpoint.windowId; StorageAgent agent = oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT); agent.save(oo, oper.id, windowId); if (agent instanceof AsyncStorageAgent) { ((AsyncStorageAgent)agent).flush(oper.id, windowId); } } catch (IOException e) { // inconsistent state, no recovery option, requires shutdown throw new IllegalStateException("Failed to write operator state after partition change " + oper, e); } oper.setRecoveryCheckpoint(checkpoint); if (!Checkpoint.INITIAL_CHECKPOINT.equals(checkpoint)) { oper.checkpoints.add(checkpoint); } }
Example #11
Source File: ApexStreamImpl.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
@Override @SuppressWarnings("unchecked") public ApexStream<T> with(DAG.Locality locality) { if (lastBrick.lastStream != null) { for (DagMeta.NodeMeta parent : lastBrick.nodeMeta.getParent()) { Pair<List<Operator.InputPort>, DAG.Locality> p = parent.getNodeStreams().get(lastBrick.lastStream.getLeft()); if (p != null) { p.setValue(locality); } } } return this; }
Example #12
Source File: AutoMetricTest.java From attic-apex-core with Apache License 2.0 | 5 votes |
@Override public OperatorResponse execute(Operator oper, int arg1, long arg2) throws IOException { if (oper instanceof TestOperator) { LOG.debug("Setting property"); ((TestOperator)oper).propVal = true; } return new TestOperatorResponse(); }
Example #13
Source File: CSVMessageFormat.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
@Override public RelInfo populateInputDAG(DAG dag, JavaTypeFactory typeFactory) { CsvParser csvParser = dag.addOperator(OperatorUtils.getUniqueOperatorName("CSVParser"), CsvParser.class); csvParser.setSchema((String)operands.get(CSV_SCHEMA)); return new RelInfo("CSVParser", Lists.<Operator.InputPort>newArrayList(csvParser.in), csvParser, csvParser.out, getRowType(typeFactory)); }
Example #14
Source File: PhysicalPlan.java From attic-apex-core with Apache License 2.0 | 5 votes |
@SuppressWarnings("deprecation") @Override public StatsListener.OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException { cmd.execute(operator,operatorId,windowId); return null; }
Example #15
Source File: PhysicalPlan.java From attic-apex-core with Apache License 2.0 | 5 votes |
private PTOperator addPTOperator(PMapping nodeDecl, Partition<? extends Operator> partition, Checkpoint checkpoint) { PTOperator oper = newOperator(nodeDecl.logicalOperator, nodeDecl.logicalOperator.getName()); oper.recoveryCheckpoint = checkpoint; // output port objects for (Map.Entry<LogicalPlan.OutputPortMeta, StreamMeta> outputEntry : nodeDecl.logicalOperator.getOutputStreams().entrySet()) { setupOutput(nodeDecl, oper, outputEntry); } String host = null; if (partition != null) { oper.setPartitionKeys(partition.getPartitionKeys()); host = partition.getAttributes().get(OperatorContext.LOCALITY_HOST); } if (host == null) { host = nodeDecl.logicalOperator.getValue(OperatorContext.LOCALITY_HOST); } nodeDecl.addPartition(oper); this.newOpers.put(oper, partition != null ? partition.getPartitionedInstance() : nodeDecl.logicalOperator.getOperator()); // // update locality // setLocalityGrouping(nodeDecl, oper, inlinePrefs, Locality.CONTAINER_LOCAL, host); setLocalityGrouping(nodeDecl, oper, localityPrefs, Locality.NODE_LOCAL, host); return oper; }
Example #16
Source File: PhysicalPlan.java From attic-apex-core with Apache License 2.0 | 5 votes |
public Operator loadOperator(PTOperator oper) { try { LOG.debug("Loading state for {}", oper); return (Operator)oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT).load(oper.id, oper.isOperatorStateLess() ? Stateless.WINDOW_ID : oper.recoveryCheckpoint.windowId); } catch (IOException e) { throw new RuntimeException("Failed to read partition state for " + oper, e); } }
Example #17
Source File: LogicalPlan.java From attic-apex-core with Apache License 2.0 | 5 votes |
public void findInvalidDelays(OperatorMeta om, List<List<String>> invalidDelays, Stack<OperatorMeta> stack) { stack.push(om); // depth first successors traversal boolean isDelayOperator = om.getOperator() instanceof Operator.DelayOperator; if (isDelayOperator) { if (om.getValue(OperatorContext.APPLICATION_WINDOW_COUNT) != 1) { LOG.debug("detected DelayOperator having APPLICATION_WINDOW_COUNT not equal to 1"); invalidDelays.add(Collections.singletonList(om.getName())); } } for (StreamMeta downStream: om.outputStreams.values()) { for (InputPortMeta sink : downStream.sinks) { OperatorMeta successor = sink.getOperatorMeta(); if (isDelayOperator) { sink.attributes.put(IS_CONNECTED_TO_DELAY_OPERATOR, true); // Check whether all downstream operators are already visited in the path if (successor != null && !stack.contains(successor)) { LOG.debug("detected DelayOperator does not immediately output to a visited operator {}.{}->{}.{}", om.getName(), downStream.getSource().getPortName(), successor.getName(), sink.getPortName()); invalidDelays.add(Arrays.asList(om.getName(), successor.getName())); } } else { findInvalidDelays(successor, invalidDelays, stack); } } } stack.pop(); }
Example #18
Source File: OperatorDiscoverer.java From attic-apex-core with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") public Class<? extends Operator> getOperatorClass(String className) throws ClassNotFoundException { if (CollectionUtils.isEmpty(operatorClassNames)) { loadGenericOperatorClasses(); } Class<?> clazz = classLoader.loadClass(className); if (!Operator.class.isAssignableFrom(clazz)) { throw new IllegalArgumentException("Argument must be a subclass of Operator class"); } return (Class<? extends Operator>)clazz; }
Example #19
Source File: Operators.java From attic-apex-core with Apache License 2.0 | 5 votes |
@Override public void addOutputPort(Operator.OutputPort<?> port, Field field, OutputPortFieldAnnotation portAnnotation, AppData.ResultPort adrAnnotation) { if (!outputPorts.containsKey(field.getName())) { outputPorts.put(field.getName(), new PortContextPair<OutputPort<?>>(port)); } }
Example #20
Source File: CreateOperatorRequest.java From attic-apex-core with Apache License 2.0 | 5 votes |
@Override public void execute(PlanModifier pm) { Class<? extends Operator> operClass = StramUtils.classForName(operatorFQCN, Operator.class); Operator operator = StramUtils.newInstance(operClass); pm.addOperator(operatorName, operator); }
Example #21
Source File: DagMeta.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
public NodeMeta addNode(Operator operator, NodeMeta parent, Operator.OutputPort parentOutput, Operator.InputPort inputPort, Option... options) { NodeMeta newNode = new NodeMeta(operator, options); if (parent == null) { heads.add(newNode); } else { parent.nodeStreams.get(parentOutput).getLeft().add(inputPort); parent.children.add(newNode); newNode.parent.add(parent); } return newNode; }
Example #22
Source File: PhysicalPlan.java From attic-apex-core with Apache License 2.0 | 5 votes |
RepartitionContext(Partitioner<Operator> partitioner, PMapping currentMapping, int partitionCount) { super(currentMapping, partitionCount); this.operators = currentMapping.partitions; this.currentPartitions = new ArrayList<>(operators.size()); this.currentPartitionMap = Maps.newHashMapWithExpectedSize(operators.size()); this.operatorIdToPartition = Maps.newHashMapWithExpectedSize(operators.size()); // collect current partitions with committed operator state // those will be needed by the partitioner for split/merge for (PTOperator pOperator : operators) { Map<InputPort<?>, PartitionKeys> pks = pOperator.getPartitionKeys(); if (pks == null) { throw new AssertionError("Null partition: " + pOperator); } // if partitions checkpoint at different windows, processing for new or modified // partitions will start from earliest checkpoint found (at least once semantics) if (minCheckpoint == null) { minCheckpoint = pOperator.recoveryCheckpoint; } else if (minCheckpoint.windowId > pOperator.recoveryCheckpoint.windowId) { minCheckpoint = pOperator.recoveryCheckpoint; } Operator partitionedOperator = loadOperator(pOperator); DefaultPartition<Operator> partition = new DefaultPartition<>(partitionedOperator, pks, pOperator.loadIndicator, pOperator.stats); currentPartitions.add(partition); currentPartitionMap.put(partition, pOperator); LOG.debug("partition load: {} {} {}", pOperator, partition.getPartitionKeys(), partition.getLoad()); operatorIdToPartition.put(pOperator.getId(), partition); } newPartitions = partitioner.definePartitions(new ArrayList<Partition<Operator>>(currentPartitions), this); }
Example #23
Source File: PropertyInjectorVisitor.java From attic-apex-core with Apache License 2.0 | 5 votes |
@Override public void handle(DAGSetupEvent event) { for (DAG.OperatorMeta ometa : dag.getAllOperatorsMeta()) { Operator o = ometa.getOperator(); LogicalPlanConfiguration.setOperatorProperties(o, propertyMap); } }
Example #24
Source File: LogicalPlan.java From attic-apex-core with Apache License 2.0 | 5 votes |
@Override public OperatorMeta getMeta(Operator operator) { // TODO: cache mapping for (OperatorMeta o: getAllOperators()) { if (o.operator == operator) { return o; } } throw new IllegalArgumentException("Operator not associated with the DAG: " + operator); }
Example #25
Source File: ApexStreamImpl.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
@Override @SuppressWarnings("unchecked") public ApexStreamImpl<T> print() { ConsoleOutputOperator consoleOutputOperator = new ConsoleOutputOperator(); addOperator(consoleOutputOperator, (Operator.InputPort<T>)consoleOutputOperator.input, null, Option.Options.name(IDGenerator.generateOperatorIDWithUUID(consoleOutputOperator.getClass()))); return this; }
Example #26
Source File: LogicalPlanTest.java From attic-apex-core with Apache License 2.0 | 5 votes |
@Test public void testAtMostOnceProcessingModeValidation() { TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class); TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class); GenericTestOperator amoOper = dag.addOperator("amoOper", GenericTestOperator.class); dag.setOperatorAttribute(amoOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_MOST_ONCE); dag.addStream("input1.outport", input1.outport, amoOper.inport1); dag.addStream("input2.outport", input2.outport, amoOper.inport2); GenericTestOperator outputOper = dag.addOperator("outputOper", GenericTestOperator.class); dag.setOperatorAttribute(outputOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_LEAST_ONCE); dag.addStream("aloOper.outport1", amoOper.outport1, outputOper.inport1); try { dag.validate(); Assert.fail("Exception expected for " + outputOper); } catch (ValidationException ve) { Assert.assertEquals("", ve.getMessage(), "Processing mode outputOper/AT_LEAST_ONCE not valid for source amoOper/AT_MOST_ONCE"); } dag.setOperatorAttribute(outputOper, OperatorContext.PROCESSING_MODE, null); dag.validate(); OperatorMeta outputOperOm = dag.getMeta(outputOper); Assert.assertEquals("" + outputOperOm.getAttributes(), Operator.ProcessingMode.AT_MOST_ONCE, outputOperOm.getValue(OperatorContext.PROCESSING_MODE)); }
Example #27
Source File: TwitterTopCounterApplication.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
public static void consoleOutput(DAG dag, String operatorName, OutputPort<List<Map<String, Object>>> topCount, String schemaFile, String alias) { if (PubSubHelper.isGatewayConfigured(dag)) { URI uri = PubSubHelper.getURI(dag); AppDataSnapshotServerMap snapshotServer = dag.addOperator("SnapshotServer", new AppDataSnapshotServerMap()); Map<String, String> conversionMap = Maps.newHashMap(); conversionMap.put(alias, WindowedTopCounter.FIELD_TYPE); String snapshotServerJSON = SchemaUtils.jarResourceFileToString(schemaFile); snapshotServer.setSnapshotSchemaJSON(snapshotServerJSON); snapshotServer.setTableFieldToMapField(conversionMap); PubSubWebSocketAppDataQuery wsQuery = new PubSubWebSocketAppDataQuery(); wsQuery.setUri(uri); snapshotServer.setEmbeddableQueryInfoProvider(wsQuery); PubSubWebSocketAppDataResult wsResult = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult()); wsResult.setUri(uri); Operator.InputPort<String> queryResultPort = wsResult.input; dag.addStream("MapProvider", topCount, snapshotServer.input); dag.addStream("Result", snapshotServer.queryResult, queryResultPort); } else { ConsoleOutputOperator operator = dag.addOperator(operatorName, new ConsoleOutputOperator()); operator.setStringFormat(operatorName + ": %s"); dag.addStream("MapProvider", topCount, operator.input); } }
Example #28
Source File: LogicalPlan.java From attic-apex-core with Apache License 2.0 | 5 votes |
@Override @SuppressWarnings("unchecked") public <T> StreamMeta addStream(String id, Operator.OutputPort<? extends T> source, Operator.InputPort<? super T> sink1) { @SuppressWarnings("rawtypes") InputPort[] ports = new Operator.InputPort[]{sink1}; return addStream(id, source, ports); }
Example #29
Source File: LogicalPlan.java From attic-apex-core with Apache License 2.0 | 5 votes |
@SuppressWarnings({"unchecked", "rawtypes"}) private void addDAGToCurrentDAG(ModuleMeta moduleMeta) { LogicalPlan subDag = moduleMeta.getDag(); String subDAGName = moduleMeta.getName(); String name; for (OperatorMeta operatorMeta : subDag.getAllOperators()) { name = subDAGName + MODULE_NAMESPACE_SEPARATOR + operatorMeta.getName(); Operator op = this.addOperator(name, operatorMeta.getOperator()); OperatorMeta operatorMetaNew = this.getMeta(op); operatorMetaNew.copyAttributesFrom(operatorMeta); operatorMetaNew.setModuleName(operatorMeta.getModuleName() == null ? subDAGName : subDAGName + MODULE_NAMESPACE_SEPARATOR + operatorMeta.getModuleName()); } for (StreamMeta streamMeta : subDag.getAllStreams()) { OutputPortMeta sourceMeta = streamMeta.getSource(); List<InputPort<?>> ports = new LinkedList<>(); for (InputPortMeta inputPortMeta : streamMeta.getSinks()) { ports.add(inputPortMeta.getPort()); } InputPort[] inputPorts = ports.toArray(new InputPort[]{}); name = subDAGName + MODULE_NAMESPACE_SEPARATOR + streamMeta.getName(); StreamMeta streamMetaNew = this.addStream(name, sourceMeta.getPort(), inputPorts); streamMetaNew.setLocality(streamMeta.getLocality()); } }
Example #30
Source File: LogicalPlan.java From attic-apex-core with Apache License 2.0 | 5 votes |
@Override public <T extends Operator> T addOperator(@Nonnull String name, Class<T> clazz) { T instance; try { instance = clazz.newInstance(); } catch (Exception ex) { throw new IllegalArgumentException(ex); } addOperator(name, instance); return instance; }