Java Code Examples for com.datatorrent.stram.plan.logical.LogicalPlan#InputPortMeta

The following examples show how to use com.datatorrent.stram.plan.logical.LogicalPlan#InputPortMeta . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StramWebServices.java    From Bats with Apache License 2.0 6 votes vote down vote up 
@GET
@Path(PATH_LOGICAL_PLAN_OPERATORS + "/{operatorName}/ports")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getPorts(@PathParam("operatorName") String operatorName)
{
  init();
  OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
  Set<LogicalPlan.InputPortMeta> inputPorts;
  Set<LogicalPlan.OutputPortMeta> outputPorts;
  if (logicalOperator == null) {
    ModuleMeta logicalModule = dagManager.getModuleMeta(operatorName);
    if (logicalModule == null) {
      throw new NotFoundException();
    }
    inputPorts = logicalModule.getInputStreams().keySet();
    outputPorts = logicalModule.getOutputStreams().keySet();
  } else {
    inputPorts = logicalOperator.getInputStreams().keySet();
    outputPorts = logicalOperator.getOutputStreams().keySet();
  }

  JSONObject result = getPortsObjects(inputPorts, outputPorts);
  return result;
}
Example 2
Source File: PhysicalPlan.java    From Bats with Apache License 2.0 6 votes vote down vote up 
/**
 * Remove the given partition with any associated parallel partitions and
 * per-partition outputStreams.
 *
 * @param oper
 * @return
 */
private void removePartition(PTOperator oper, PMapping operatorMapping)
{
  // remove any parallel partition
  for (PTOutput out : oper.outputs) {
    // copy list as it is modified by recursive remove
    for (PTInput in : Lists.newArrayList(out.sinks)) {
      for (LogicalPlan.InputPortMeta im : in.logicalStream.getSinks()) {
        PMapping m = this.logicalToPTOperator.get(im.getOperatorMeta());
        if (m.parallelPartitions == operatorMapping.parallelPartitions) {
          // associated operator parallel partitioned
          removePartition(in.target, operatorMapping);
          m.partitions.remove(in.target);
        }
      }
    }
  }
  // remove the operator
  removePTOperator(oper);
}
Example 3
Source File: StramWebServices.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up 
@GET
@Path(PATH_LOGICAL_PLAN_OPERATORS + "/{operatorName}/ports")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getPorts(@PathParam("operatorName") String operatorName)
{
  init();
  OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
  Set<LogicalPlan.InputPortMeta> inputPorts;
  Set<LogicalPlan.OutputPortMeta> outputPorts;
  if (logicalOperator == null) {
    ModuleMeta logicalModule = dagManager.getModuleMeta(operatorName);
    if (logicalModule == null) {
      throw new NotFoundException();
    }
    inputPorts = logicalModule.getInputStreams().keySet();
    outputPorts = logicalModule.getOutputStreams().keySet();
  } else {
    inputPorts = logicalOperator.getInputStreams().keySet();
    outputPorts = logicalOperator.getOutputStreams().keySet();
  }

  JSONObject result = getPortsObjects(inputPorts, outputPorts);
  return result;
}
Example 4
Source File: PhysicalPlan.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up 
/**
 * Remove the given partition with any associated parallel partitions and
 * per-partition outputStreams.
 *
 * @param oper
 * @return
 */
private void removePartition(PTOperator oper, PMapping operatorMapping)
{
  // remove any parallel partition
  for (PTOutput out : oper.outputs) {
    // copy list as it is modified by recursive remove
    for (PTInput in : Lists.newArrayList(out.sinks)) {
      for (LogicalPlan.InputPortMeta im : in.logicalStream.getSinks()) {
        PMapping m = this.logicalToPTOperator.get(im.getOperatorMeta());
        if (m.parallelPartitions == operatorMapping.parallelPartitions) {
          // associated operator parallel partitioned
          removePartition(in.target, operatorMapping);
          m.partitions.remove(in.target);
        }
      }
    }
  }
  // remove the operator
  removePTOperator(oper);
}
Example 5
Source File: TestModuleExpansion.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up 
private void validateSeperateStream(LogicalPlan dag, String streamName, String inputOperatorName,
    String... outputOperatorNames)
{
  LogicalPlan.StreamMeta streamMeta = dag.getStream(streamName);
  String sourceName = streamMeta.getSource().getOperatorMeta().getName();

  List<String> sinksName = new ArrayList<>();
  for (LogicalPlan.InputPortMeta inputPortMeta : streamMeta.getSinks()) {
    sinksName.add(inputPortMeta.getOperatorMeta().getName());
  }

  Assert.assertTrue(inputOperatorName.equals(sourceName));
  Assert.assertEquals(outputOperatorNames.length, sinksName.size());

  for (String outputOperatorName : outputOperatorNames) {
    Assert.assertTrue(sinksName.contains(outputOperatorName));
  }
}
Example 6
Source File: StramWebServices.java    From Bats with Apache License 2.0 5 votes vote down vote up 
@GET
@Path(PATH_LOGICAL_PLAN_OPERATORS + "/{operatorName}/ports/{portName}")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getPort(@PathParam("operatorName") String operatorName, @PathParam("portName") String portName)
{
  init();
  OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
  Set<LogicalPlan.InputPortMeta> inputPorts;
  Set<LogicalPlan.OutputPortMeta> outputPorts;
  if (logicalOperator == null) {
    ModuleMeta logicalModule = dagManager.getModuleMeta(operatorName);
    if (logicalModule == null) {
      throw new NotFoundException();
    }
    inputPorts = logicalModule.getInputStreams().keySet();
    outputPorts = logicalModule.getOutputStreams().keySet();
  } else {
    inputPorts = logicalOperator.getInputStreams().keySet();
    outputPorts = logicalOperator.getOutputStreams().keySet();
  }

  try {
    JSONObject resp = getPortObject(inputPorts, outputPorts, portName);
    if (resp != null) {
      return resp;
    }
  } catch (JSONException ex) {
    throw new RuntimeException(ex);
  }
  throw new NotFoundException();
}
Example 7
Source File: StramWebServices.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up 
@GET
@Path(PATH_LOGICAL_PLAN_OPERATORS + "/{operatorName}/ports/{portName}")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getPort(@PathParam("operatorName") String operatorName, @PathParam("portName") String portName)
{
  init();
  OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
  Set<LogicalPlan.InputPortMeta> inputPorts;
  Set<LogicalPlan.OutputPortMeta> outputPorts;
  if (logicalOperator == null) {
    ModuleMeta logicalModule = dagManager.getModuleMeta(operatorName);
    if (logicalModule == null) {
      throw new NotFoundException();
    }
    inputPorts = logicalModule.getInputStreams().keySet();
    outputPorts = logicalModule.getOutputStreams().keySet();
  } else {
    inputPorts = logicalOperator.getInputStreams().keySet();
    outputPorts = logicalOperator.getOutputStreams().keySet();
  }

  try {
    JSONObject resp = getPortObject(inputPorts, outputPorts, portName);
    if (resp != null) {
      return resp;
    }
  } catch (JSONException ex) {
    throw new RuntimeException(ex);
  }
  throw new NotFoundException();
}
Example 8
Source File: StreamCodecTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up 
private OperatorDeployInfo.InputDeployInfo getInputDeployInfo(OperatorDeployInfo odi, LogicalPlan.InputPortMeta
    portMeta)
{
  OperatorDeployInfo.InputDeployInfo idi = null;
  List<OperatorDeployInfo.InputDeployInfo> inputs = odi.inputs;
  for (OperatorDeployInfo.InputDeployInfo input : inputs) {
    if (input.portName.equals(portMeta.getPortName())) {
      idi = input;
      break;
    }
  }
  Assert.assertNotNull("input deploy info " + portMeta.getPortName(), idi);
  return idi;
}
Example 9
Source File: PhysicalPlan.java    From Bats with Apache License 2.0 4 votes vote down vote up 
/**
 * Add logical operator to the plan. Assumes that upstream operators have been added before.
 * @param om
 */
public final void addLogicalOperator(OperatorMeta om)
{
  PMapping pnodes = new PMapping(om);
  String host = pnodes.logicalOperator.getValue(OperatorContext.LOCALITY_HOST);
  localityPrefs.add(pnodes, host);

  PMapping upstreamPartitioned = null;

  for (Map.Entry<LogicalPlan.InputPortMeta, StreamMeta> e : om.getInputStreams().entrySet()) {
    if (e.getValue().getSource().getOperatorMeta().getOperator() instanceof Operator.DelayOperator) {
      continue;
    }
    PMapping m = logicalToPTOperator.get(e.getValue().getSource().getOperatorMeta());
    if (e.getKey().getValue(PortContext.PARTITION_PARALLEL).equals(true)) {
      // operator partitioned with upstream
      if (upstreamPartitioned != null) {
        // need to have common root
        if (!upstreamPartitioned.parallelPartitions.contains(m.logicalOperator) && upstreamPartitioned != m) {
          String msg = String.format("operator cannot extend multiple partitions (%s and %s)", upstreamPartitioned.logicalOperator, m.logicalOperator);
          throw new AssertionError(msg);
        }
      }
      m.parallelPartitions.add(pnodes.logicalOperator);
      pnodes.parallelPartitions = m.parallelPartitions;
      upstreamPartitioned = m;
    }

    if (Locality.CONTAINER_LOCAL == e.getValue().getLocality() || Locality.THREAD_LOCAL == e.getValue().getLocality()) {
      inlinePrefs.setLocal(m, pnodes);
    } else if (Locality.NODE_LOCAL == e.getValue().getLocality()) {
      localityPrefs.setLocal(m, pnodes);
    }
  }

  //
  // create operator instances
  //
  this.logicalToPTOperator.put(om, pnodes);
  if (upstreamPartitioned != null) {
    // parallel partition
    //LOG.debug("Operator {} should be partitioned to {} partitions", pnodes.logicalOperator.getName(), upstreamPartitioned.partitions.size());
    initPartitioning(pnodes, upstreamPartitioned.partitions.size());
  } else {
    initPartitioning(pnodes, 0);
  }
  updateStreamMappings(pnodes);
}
Example 10
Source File: PhysicalPlan.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up 
/**
 * Add logical operator to the plan. Assumes that upstream operators have been added before.
 * @param om
 */
public final void addLogicalOperator(OperatorMeta om)
{
  PMapping pnodes = new PMapping(om);
  String host = pnodes.logicalOperator.getValue(OperatorContext.LOCALITY_HOST);
  localityPrefs.add(pnodes, host);

  PMapping upstreamPartitioned = null;

  for (Map.Entry<LogicalPlan.InputPortMeta, StreamMeta> e : om.getInputStreams().entrySet()) {
    if (e.getValue().getSource().getOperatorMeta().getOperator() instanceof Operator.DelayOperator) {
      continue;
    }
    PMapping m = logicalToPTOperator.get(e.getValue().getSource().getOperatorMeta());
    if (e.getKey().getValue(PortContext.PARTITION_PARALLEL).equals(true)) {
      // operator partitioned with upstream
      if (upstreamPartitioned != null) {
        // need to have common root
        if (!upstreamPartitioned.parallelPartitions.contains(m.logicalOperator) && upstreamPartitioned != m) {
          String msg = String.format("operator cannot extend multiple partitions (%s and %s)", upstreamPartitioned.logicalOperator, m.logicalOperator);
          throw new AssertionError(msg);
        }
      }
      m.parallelPartitions.add(pnodes.logicalOperator);
      pnodes.parallelPartitions = m.parallelPartitions;
      upstreamPartitioned = m;
    }

    if (Locality.CONTAINER_LOCAL == e.getValue().getLocality() || Locality.THREAD_LOCAL == e.getValue().getLocality()) {
      inlinePrefs.setLocal(m, pnodes);
    } else if (Locality.NODE_LOCAL == e.getValue().getLocality()) {
      localityPrefs.setLocal(m, pnodes);
    }
  }

  //
  // create operator instances
  //
  this.logicalToPTOperator.put(om, pnodes);
  if (upstreamPartitioned != null) {
    // parallel partition
    //LOG.debug("Operator {} should be partitioned to {} partitions", pnodes.logicalOperator.getName(), upstreamPartitioned.partitions.size());
    initPartitioning(pnodes, upstreamPartitioned.partitions.size());
  } else {
    initPartitioning(pnodes, 0);
  }
  updateStreamMappings(pnodes);
}
Example 11
Source File: TestModuleExpansion.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up 
/**
 * Validate attribute set on the port of DummyOperator in Level1Module
 */
private void validatePortAttribute(LogicalPlan dag, String name, int memory)
{
  LogicalPlan.InputPortMeta imeta = dag.getOperatorMeta(name).getInputStreams().keySet().iterator().next();
  Assert.assertEquals(memory, (int)imeta.getAttributes().get(Context.PortContext.BUFFER_MEMORY_MB));
}