Java Code Examples for com.datatorrent.api.Operator#DelayOperator

@Override
public StreamMeta addSink(Operator.InputPort<?> port)
{
  if (port instanceof ProxyInputPort) {
    proxySinks.add(port);
    return this;
  }
  InputPortMeta portMeta = assertGetPortMeta(port);
  OperatorMeta om = portMeta.getOperatorMeta();
  String portName = portMeta.getPortName();
  if (om.inputStreams.containsKey(portMeta)) {
    throw new IllegalArgumentException(String.format("Port %s already connected to stream %s", portName, om.inputStreams.get(portMeta)));
  }

  /*
  finalizeValidate(portMeta);
  */

  sinks.add(portMeta);
  om.inputStreams.put(portMeta, this);
  rootOperators.remove(portMeta.operatorMeta);
  if (this.source != null && !(this.source.getOperatorMeta().getOperator() instanceof Operator.DelayOperator)) {
    leafOperators.remove(this.source.getOperatorMeta());
  }
  return this;
}
 

protected void emitEndWindow()
{
  long windowId = (operator instanceof Operator.DelayOperator) ?
      WindowGenerator.getAheadWindowId(currentWindowId, firstWindowMillis, windowWidthMillis, 1) : currentWindowId;
  EndWindowTuple ewt = new EndWindowTuple(windowId);
  for (int s = sinks.length; s-- > 0; ) {
    sinks[s].put(ewt);
  }
  controlTupleCount++;
}
 

public void remove()
{
  for (InputPortMeta ipm : this.sinks) {
    ipm.getOperatorMeta().inputStreams.remove(ipm);
    if (ipm.getOperatorMeta().inputStreams.isEmpty()) {
      rootOperators.add(ipm.getOperatorMeta());
    }
  }
  // Remove persist operator for at stream level if present:
  if (getPersistOperator() != null) {
    removeOperator(getPersistOperator().getOperator());
  }

  // Remove persist operators added for specific sinks :
  if (!sinkSpecificPersistOperatorMap.isEmpty()) {
    for (Entry<InputPortMeta, OperatorMeta> entry : sinkSpecificPersistOperatorMap.entrySet()) {
      removeOperator(entry.getValue().getOperator());
    }
    sinkSpecificPersistOperatorMap.clear();
    sinkSpecificPersistInputPortMap.clear();
  }
  this.sinks.clear();
  if (this.source != null) {
    this.source.getOperatorMeta().outputStreams.remove(this.source);
    if (this.source.getOperatorMeta().outputStreams.isEmpty() &&
        !(this.source.getOperatorMeta().getOperator() instanceof Operator.DelayOperator)) {
      leafOperators.remove(this.source.getOperatorMeta());
    }
  }
  this.source = null;
  streams.remove(this.id);
}
 

public void findInvalidDelays(OperatorMeta om, List<List<String>> invalidDelays, Stack<OperatorMeta> stack)
{
  stack.push(om);

  // depth first successors traversal
  boolean isDelayOperator = om.getOperator() instanceof Operator.DelayOperator;
  if (isDelayOperator) {
    if (om.getValue(OperatorContext.APPLICATION_WINDOW_COUNT) != 1) {
      LOG.debug("detected DelayOperator having APPLICATION_WINDOW_COUNT not equal to 1");
      invalidDelays.add(Collections.singletonList(om.getName()));
    }
  }

  for (StreamMeta downStream: om.outputStreams.values()) {
    for (InputPortMeta sink : downStream.sinks) {
      OperatorMeta successor = sink.getOperatorMeta();
      if (isDelayOperator) {
        sink.attributes.put(IS_CONNECTED_TO_DELAY_OPERATOR, true);
        // Check whether all downstream operators are already visited in the path
        if (successor != null && !stack.contains(successor)) {
          LOG.debug("detected DelayOperator does not immediately output to a visited operator {}.{}->{}.{}",
              om.getName(), downStream.getSource().getPortName(), successor.getName(), sink.getPortName());
          invalidDelays.add(Arrays.asList(om.getName(), successor.getName()));
        }
      } else {
        findInvalidDelays(successor, invalidDelays, stack);
      }
    }
  }
  stack.pop();
}
 

/**
 * Check for cycles in the graph reachable from start node n. This is done by
 * attempting to find strongly connected components.
 *
 * @see <a href="http://en.wikipedia.org/wiki/Tarjan%E2%80%99s_strongly_connected_components_algorithm">http://en.wikipedia.org/wiki/Tarjan%E2%80%99s_strongly_connected_components_algorithm</a>
 *
 * @param om
 * @param ctx
 */
public void findStronglyConnected(OperatorMeta om, ValidationContext ctx)
{
  om.nindex = ctx.nodeIndex;
  om.lowlink = ctx.nodeIndex;
  ctx.nodeIndex++;
  ctx.stack.push(om);
  ctx.path.push(om);

  // depth first successors traversal
  for (StreamMeta downStream: om.outputStreams.values()) {
    for (InputPortMeta sink: downStream.sinks) {
      OperatorMeta successor = sink.getOperatorMeta();
      if (successor == null) {
        continue;
      }
      // check for self referencing node
      if (om == successor) {
        ctx.invalidCycles.add(Collections.singleton(om));
      }
      if (successor.nindex == null) {
        // not visited yet
        findStronglyConnected(successor, ctx);
        om.lowlink = Math.min(om.lowlink, successor.lowlink);
      } else if (ctx.stack.contains(successor)) {
        om.lowlink = Math.min(om.lowlink, successor.nindex);
        boolean isDelayLoop = false;
        for (int i = ctx.stack.size(); i > 0; i--) {
          OperatorMeta om2 = ctx.stack.get(i - 1);
          if (om2.getOperator() instanceof Operator.DelayOperator) {
            isDelayLoop = true;
          }
          if (om2 == successor) {
            break;
          }
        }
        if (!isDelayLoop) {
          ctx.invalidLoopAt = successor;
        }
      }
    }
  }

  // pop stack for all root operators
  if (om.lowlink.equals(om.nindex)) {
    Set<OperatorMeta> connectedSet = new LinkedHashSet<>(ctx.stack.size());
    while (!ctx.stack.isEmpty()) {
      OperatorMeta n2 = ctx.stack.pop();
      connectedSet.add(n2);
      if (n2 == om) {
        break; // collected all connected operators
      }
    }
    // strongly connected (cycle) if more than one node in stack
    if (connectedSet.size() > 1) {
      ctx.stronglyConnected.add(connectedSet);
      if (connectedSet.contains(ctx.invalidLoopAt)) {
        ctx.invalidCycles.add(connectedSet);
      }
    }
  }
  ctx.path.pop();

}
 

/**
 * Add logical operator to the plan. Assumes that upstream operators have been added before.
 * @param om
 */
public final void addLogicalOperator(OperatorMeta om)
{
  PMapping pnodes = new PMapping(om);
  String host = pnodes.logicalOperator.getValue(OperatorContext.LOCALITY_HOST);
  localityPrefs.add(pnodes, host);

  PMapping upstreamPartitioned = null;

  for (Map.Entry<LogicalPlan.InputPortMeta, StreamMeta> e : om.getInputStreams().entrySet()) {
    if (e.getValue().getSource().getOperatorMeta().getOperator() instanceof Operator.DelayOperator) {
      continue;
    }
    PMapping m = logicalToPTOperator.get(e.getValue().getSource().getOperatorMeta());
    if (e.getKey().getValue(PortContext.PARTITION_PARALLEL).equals(true)) {
      // operator partitioned with upstream
      if (upstreamPartitioned != null) {
        // need to have common root
        if (!upstreamPartitioned.parallelPartitions.contains(m.logicalOperator) && upstreamPartitioned != m) {
          String msg = String.format("operator cannot extend multiple partitions (%s and %s)", upstreamPartitioned.logicalOperator, m.logicalOperator);
          throw new AssertionError(msg);
        }
      }
      m.parallelPartitions.add(pnodes.logicalOperator);
      pnodes.parallelPartitions = m.parallelPartitions;
      upstreamPartitioned = m;
    }

    if (Locality.CONTAINER_LOCAL == e.getValue().getLocality() || Locality.THREAD_LOCAL == e.getValue().getLocality()) {
      inlinePrefs.setLocal(m, pnodes);
    } else if (Locality.NODE_LOCAL == e.getValue().getLocality()) {
      localityPrefs.setLocal(m, pnodes);
    }
  }

  //
  // create operator instances
  //
  this.logicalToPTOperator.put(om, pnodes);
  if (upstreamPartitioned != null) {
    // parallel partition
    //LOG.debug("Operator {} should be partitioned to {} partitions", pnodes.logicalOperator.getName(), upstreamPartitioned.partitions.size());
    initPartitioning(pnodes, upstreamPartitioned.partitions.size());
  } else {
    initPartitioning(pnodes, 0);
  }
  updateStreamMappings(pnodes);
}