org.apache.flink.optimizer.plan.PlanNode Java Examples

/**
 * Verify operator parallelism.
 *
 * @param env the Flink execution environment.
 * @param expectedParallelism expected operator parallelism
 */
public static void verifyParallelism(ExecutionEnvironment env, int expectedParallelism) {
	env.setParallelism(2 * expectedParallelism);

	Optimizer compiler = new Optimizer(null, new DefaultCostEstimator(), new Configuration());
	OptimizedPlan optimizedPlan = compiler.compile(env.createProgramPlan());

	List<PlanNode> queue = new ArrayList<>();
	queue.addAll(optimizedPlan.getDataSinks());

	while (queue.size() > 0) {
		PlanNode node = queue.remove(queue.size() - 1);

		// Data sources may have parallelism of 1, so simply check that the node
		// parallelism has not been increased by setting the default parallelism
		assertTrue("Wrong parallelism for " + node.toString(), node.getParallelism() <= expectedParallelism);

		for (Channel channel : node.getInputs()) {
			queue.add(channel.getSource());
		}
	}
}
 

protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
		List<PlanNode> target, CostEstimator estimator)
{
	for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
		final Channel in = template.clone();
		ilp.parameterizeChannel(in);
		
		// instantiate a candidate, if the instantiated local properties meet one possible local property set
		outer:
		for (OperatorDescriptorSingle dps: getPossibleProperties()) {
			for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
				if (ilps.isMetBy(in.getLocalProperties())) {
					in.setRequiredLocalProps(ilps);
					instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
					break outer;
				}
			}
		}
	}
}
 

/**
 * Checks whether to candidate plans for the sub-plan of this node are comparable. The two
 * alternative plans are comparable, if
 * 
 * a) There is no branch in the sub-plan of this node
 * b) Both candidates have the same candidate as the child at the last open branch. 
 * 
 * @param plan1 The root node of the first candidate plan.
 * @param plan2 The root node of the second candidate plan.
 * @return True if the nodes are branch compatible in the inputs.
 */
protected boolean areBranchCompatible(PlanNode plan1, PlanNode plan2) {
	if (plan1 == null || plan2 == null) {
		throw new NullPointerException();
	}
	
	// if there is no open branch, the children are always compatible.
	// in most plans, that will be the dominant case
	if (this.hereJoinedBranches == null || this.hereJoinedBranches.isEmpty()) {
		return true;
	}

	for (OptimizerNode joinedBrancher : hereJoinedBranches) {
		final PlanNode branch1Cand = plan1.getCandidateAtBranchPoint(joinedBrancher);
		final PlanNode branch2Cand = plan2.getCandidateAtBranchPoint(joinedBrancher);
		
		if (branch1Cand != null && branch2Cand != null && branch1Cand != branch2Cand) {
			return false;
		}
	}
	return true;
}
 

protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
		List<PlanNode> target, CostEstimator estimator)
{
	for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
		final Channel in = template.clone();
		ilp.parameterizeChannel(in);
		
		// instantiate a candidate, if the instantiated local properties meet one possible local property set
		outer:
		for (OperatorDescriptorSingle dps: getPossibleProperties()) {
			for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
				if (ilps.isMetBy(in.getLocalProperties())) {
					in.setRequiredLocalProps(ilps);
					instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
					break outer;
				}
			}
		}
	}
}
 

protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
		List<PlanNode> target, CostEstimator estimator)
{
	for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
		final Channel in = template.clone();
		ilp.parameterizeChannel(in);
		
		// instantiate a candidate, if the instantiated local properties meet one possible local property set
		outer:
		for (OperatorDescriptorSingle dps: getPossibleProperties()) {
			for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
				if (ilps.isMetBy(in.getLocalProperties())) {
					in.setRequiredLocalProps(ilps);
					instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
					break outer;
				}
			}
		}
	}
}
 

@SuppressWarnings("unchecked")
public <T extends PlanNode> T getNode(String name, Class<? extends Function> stubClass) {
	List<PlanNode> nodes = this.map.get(name);
	if (nodes == null || nodes.isEmpty()) {
		throw new RuntimeException("No node found with the given name and stub class.");
	} else {
		PlanNode found = null;
		for (PlanNode node : nodes) {
			if (node.getClass() == stubClass) {
				if (found == null) {
					found = node;
				} else {
					throw new RuntimeException("Multiple nodes found with the given name and stub class.");
				}
			}
		}
		if (found == null) {
			throw new RuntimeException("No node found with the given name and stub class.");
		} else {
			return (T) found;
		}
	}
}
 

/**
 * Checks whether to candidate plans for the sub-plan of this node are comparable. The two
 * alternative plans are comparable, if
 * 
 * a) There is no branch in the sub-plan of this node
 * b) Both candidates have the same candidate as the child at the last open branch. 
 * 
 * @param plan1 The root node of the first candidate plan.
 * @param plan2 The root node of the second candidate plan.
 * @return True if the nodes are branch compatible in the inputs.
 */
protected boolean areBranchCompatible(PlanNode plan1, PlanNode plan2) {
	if (plan1 == null || plan2 == null) {
		throw new NullPointerException();
	}
	
	// if there is no open branch, the children are always compatible.
	// in most plans, that will be the dominant case
	if (this.hereJoinedBranches == null || this.hereJoinedBranches.isEmpty()) {
		return true;
	}

	for (OptimizerNode joinedBrancher : hereJoinedBranches) {
		final PlanNode branch1Cand = plan1.getCandidateAtBranchPoint(joinedBrancher);
		final PlanNode branch2Cand = plan2.getCandidateAtBranchPoint(joinedBrancher);
		
		if (branch1Cand != null && branch2Cand != null && branch1Cand != branch2Cand) {
			return false;
		}
	}
	return true;
}
 

@SuppressWarnings("unchecked")
public <T extends PlanNode> T getNode(String name, Class<? extends Function> stubClass) {
	List<PlanNode> nodes = this.map.get(name);
	if (nodes == null || nodes.isEmpty()) {
		throw new RuntimeException("No node found with the given name and stub class.");
	} else {
		PlanNode found = null;
		for (PlanNode node : nodes) {
			if (node.getClass() == stubClass) {
				if (found == null) {
					found = node;
				} else {
					throw new RuntimeException("Multiple nodes found with the given name and stub class.");
				}
			}
		}
		if (found == null) {
			throw new RuntimeException("No node found with the given name and stub class.");
		} else {
			return (T) found;
		}
	}
}
 

/**
 * Verify operator parallelism.
 *
 * @param env the Flink execution environment.
 * @param expectedParallelism expected operator parallelism
 */
public static void verifyParallelism(ExecutionEnvironment env, int expectedParallelism) {
	env.setParallelism(2 * expectedParallelism);

	Optimizer compiler = new Optimizer(null, new DefaultCostEstimator(), new Configuration());
	OptimizedPlan optimizedPlan = compiler.compile(env.createProgramPlan());

	List<PlanNode> queue = new ArrayList<>();
	queue.addAll(optimizedPlan.getDataSinks());

	while (queue.size() > 0) {
		PlanNode node = queue.remove(queue.size() - 1);

		// Data sources may have parallelism of 1, so simply check that the node
		// parallelism has not been increased by setting the default parallelism
		assertTrue("Wrong parallelism for " + node.toString(), node.getParallelism() <= expectedParallelism);

		for (Channel channel : node.getInputs()) {
			queue.add(channel.getSource());
		}
	}
}
 

/**
 * Verify operator parallelism.
 *
 * @param env the Flink execution environment.
 * @param expectedParallelism expected operator parallelism
 */
public static void verifyParallelism(ExecutionEnvironment env, int expectedParallelism) {
	env.setParallelism(2 * expectedParallelism);

	Optimizer compiler = new Optimizer(null, new DefaultCostEstimator(), new Configuration());
	OptimizedPlan optimizedPlan = compiler.compile(env.createProgramPlan());

	List<PlanNode> queue = new ArrayList<>();
	queue.addAll(optimizedPlan.getDataSinks());

	while (queue.size() > 0) {
		PlanNode node = queue.remove(queue.size() - 1);

		// Data sources may have parallelism of 1, so simply check that the node
		// parallelism has not been increased by setting the default parallelism
		assertTrue("Wrong parallelism for " + node.toString(), node.getParallelism() <= expectedParallelism);

		for (Channel channel : node.getInputs()) {
			queue.add(channel.getSource());
		}
	}
}
 

@Override
public boolean preVisit(PlanNode visitable) {
	if (this.seenBefore.add(visitable)) {
		if (visitable instanceof IterationPlanNode) {
			((IterationPlanNode) visitable).acceptForStepFunction(this);
		}
		return true;
	} else {
		return false;
	}
}
 

public static TypeComparatorFactory<?> getShipComparator(Channel channel, ExecutionConfig executionConfig) {
	PlanNode source = channel.getSource();
	Operator<?> javaOp = source.getProgramOperator();
	TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType();
	return createComparator(type, channel.getShipStrategyKeys(),
		getSortOrders(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()), executionConfig);
}
 

private void assignDriverResources(PlanNode node, TaskConfig config) {
	final double relativeMem = node.getRelativeMemoryPerSubTask();
	if (relativeMem > 0) {
		config.setRelativeMemoryDriver(relativeMem);
		config.setFilehandlesDriver(this.defaultMaxFan);
		config.setSpillingThresholdDriver(this.defaultSortSpillingThreshold);
	}
}
 

private boolean checkAndConfigurePersistentIntermediateResult(PlanNode node) {
	if (!(node instanceof SinkPlanNode)) {
		return false;
	}

	final Object userCodeObject = node.getProgramOperator().getUserCodeWrapper().getUserCodeObject();
	if (!(userCodeObject instanceof BlockingShuffleOutputFormat)) {
		return false;
	}

	final Iterator<Channel> inputIterator = node.getInputs().iterator();
	checkState(inputIterator.hasNext(), "SinkPlanNode must have a input.");

	final PlanNode predecessorNode = inputIterator.next().getSource();
	final JobVertex predecessorVertex = (vertices.containsKey(predecessorNode)) ?
		vertices.get(predecessorNode) :
		chainedTasks.get(predecessorNode).getContainingVertex();

	checkState(predecessorVertex != null, "Bug: Chained task has not been assigned its containing vertex when connecting.");

	predecessorVertex.createAndAddResultDataSet(
			// use specified intermediateDataSetID
			new IntermediateDataSetID(((BlockingShuffleOutputFormat) userCodeObject).getIntermediateDataSetId()),
			ResultPartitionType.BLOCKING_PERSISTENT);

	// remove this node so the OutputFormatVertex will not shown in the final JobGraph.
	vertices.remove(node);
	return true;
}
 

TaskInChain(PlanNode planNode, Class<? extends ChainedDriver<?, ?>> chainedTask,
			TaskConfig taskConfig, String taskName) {
	
	this.planNode = planNode;
	this.chainedTask = chainedTask;
	this.taskConfig = taskConfig;
	this.taskName = taskName;
}
 

@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
	if (this.cachedPlans != null) {
		return this.cachedPlans;
	} else {
		throw new IllegalStateException();
	}
}
 

public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
	if (this.cachedPlans != null) {
		throw new IllegalStateException();
	} else {
		WorksetPlanNode wspn = new WorksetPlanNode(this, "Workset ("+this.getOperator().getName()+")", gProps, lProps, initialInput);
		this.cachedPlans = Collections.<PlanNode>singletonList(wspn);
	}
}
 

private void assignDriverResources(PlanNode node, TaskConfig config) {
	final double relativeMem = node.getRelativeMemoryPerSubTask();
	if (relativeMem > 0) {
		config.setRelativeMemoryDriver(relativeMem);
		config.setFilehandlesDriver(this.defaultMaxFan);
		config.setSpillingThresholdDriver(this.defaultSortSpillingThreshold);
	}
}
 

TaskInChain(PlanNode planNode, Class<? extends ChainedDriver<?, ?>> chainedTask,
			TaskConfig taskConfig, String taskName) {
	
	this.planNode = planNode;
	this.chainedTask = chainedTask;
	this.taskConfig = taskConfig;
	this.taskName = taskName;
}
 

@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
	if (this.cachedPlans != null) {
		return this.cachedPlans;
	} else {
		throw new IllegalStateException();
	}
}
 

@Override
public void postVisit(PlanNode node) {

	if(node instanceof IterationPlanNode) {
		IterationPlanNode iNode = (IterationPlanNode)node;
		if(!visitedIterationNodes.contains(iNode)) {
			visitedIterationNodes.add(iNode);
			iNode.acceptForStepFunction(this);
		}
	}

	final Iterable<Channel> inputChannels = node.getInputs();
	for (Channel channel : inputChannels) {
		ShipStrategyType shipStrategy = channel.getShipStrategy();
		// Make sure we only optimize the DAG for range partition, and do not optimize multi times.
		if (shipStrategy == ShipStrategyType.PARTITION_RANGE) {

			if(channel.getDataDistribution() == null) {
				if (node.isOnDynamicPath()) {
					throw new InvalidProgramException("Range Partitioning not supported within iterations if users do not supply the data distribution.");
				}

				PlanNode channelSource = channel.getSource();
				List<Channel> newSourceOutputChannels = rewriteRangePartitionChannel(channel);
				channelSource.getOutgoingChannels().remove(channel);
				channelSource.getOutgoingChannels().addAll(newSourceOutputChannels);
			}
		}
	}
}
 

public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
	if (this.cachedPlans != null) {
		throw new IllegalStateException();
	} else {
		this.cachedPlans = Collections.<PlanNode>singletonList(new BulkPartialSolutionPlanNode(this,
				"PartialSolution ("+this.getOperator().getName()+")", gProps, lProps, initialInput));
	}
}
 

public static TypeComparatorFactory<?> getShipComparator(Channel channel, ExecutionConfig executionConfig) {
	PlanNode source = channel.getSource();
	Operator<?> javaOp = source.getProgramOperator();
	TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType();
	return createComparator(type, channel.getShipStrategyKeys(),
		getSortOrders(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()), executionConfig);
}
 

@Override
public void postVisit(PlanNode node) {

	if(node instanceof IterationPlanNode) {
		IterationPlanNode iNode = (IterationPlanNode)node;
		if(!visitedIterationNodes.contains(iNode)) {
			visitedIterationNodes.add(iNode);
			iNode.acceptForStepFunction(this);
		}
	}

	final Iterable<Channel> inputChannels = node.getInputs();
	for (Channel channel : inputChannels) {
		ShipStrategyType shipStrategy = channel.getShipStrategy();
		// Make sure we only optimize the DAG for range partition, and do not optimize multi times.
		if (shipStrategy == ShipStrategyType.PARTITION_RANGE) {

			if(channel.getDataDistribution() == null) {
				if (node.isOnDynamicPath()) {
					throw new InvalidProgramException("Range Partitioning not supported within iterations if users do not supply the data distribution.");
				}

				PlanNode channelSource = channel.getSource();
				List<Channel> newSourceOutputChannels = rewriteRangePartitionChannel(channel);
				channelSource.getOutgoingChannels().remove(channel);
				channelSource.getOutgoingChannels().addAll(newSourceOutputChannels);
			}
		}
	}
}
 

/**
 * Creates a new plan finalizer.
 */
public PlanFinalizer() {
	this.allNodes = new HashSet<PlanNode>();
	this.sources = new ArrayList<SourcePlanNode>();
	this.sinks = new ArrayList<SinkPlanNode>();
	this.stackOfIterationNodes = new ArrayDeque<IterationPlanNode>();
}
 

@Override
public boolean preVisit(PlanNode visitable) {
	if (this.seenBefore.add(visitable)) {
		if (visitable instanceof IterationPlanNode) {
			((IterationPlanNode) visitable).acceptForStepFunction(this);
		}
		return true;
	} else {
		return false;
	}
}
 

@SuppressWarnings("unchecked")
public <T extends PlanNode> T getNode(String name) {
	List<PlanNode> nodes = this.map.get(name);
	if (nodes == null || nodes.isEmpty()) {
		throw new RuntimeException("No node found with the given name.");
	} else if (nodes.size() != 1) {
		throw new RuntimeException("Multiple nodes found with the given name.");
	} else {
		return (T) nodes.get(0);
	}
}
 

public List<PlanNode> getNodes(String name) {
	List<PlanNode> nodes = this.map.get(name);
	if (nodes == null || nodes.isEmpty()) {
		throw new RuntimeException("No node found with the given name.");
	} else {
		return new ArrayList<PlanNode>(nodes);
	}
}
 

private void assignDriverResources(PlanNode node, TaskConfig config) {
	final double relativeMem = node.getRelativeMemoryPerSubTask();
	if (relativeMem > 0) {
		config.setRelativeMemoryDriver(relativeMem);
		config.setFilehandlesDriver(this.defaultMaxFan);
		config.setSpillingThresholdDriver(this.defaultSortSpillingThreshold);
	}
}
 

private boolean checkAndConfigurePersistentIntermediateResult(PlanNode node) {
	if (!(node instanceof SinkPlanNode)) {
		return false;
	}

	final Object userCodeObject = node.getProgramOperator().getUserCodeWrapper().getUserCodeObject();
	if (!(userCodeObject instanceof BlockingShuffleOutputFormat)) {
		return false;
	}

	final Iterator<Channel> inputIterator = node.getInputs().iterator();
	checkState(inputIterator.hasNext(), "SinkPlanNode must have a input.");

	final PlanNode predecessorNode = inputIterator.next().getSource();
	final JobVertex predecessorVertex = (vertices.containsKey(predecessorNode)) ?
		vertices.get(predecessorNode) :
		chainedTasks.get(predecessorNode).getContainingVertex();

	checkState(predecessorVertex != null, "Bug: Chained task has not been assigned its containing vertex when connecting.");

	predecessorVertex.createAndAddResultDataSet(
			// use specified intermediateDataSetID
			new IntermediateDataSetID(((BlockingShuffleOutputFormat) userCodeObject).getIntermediateDataSetId()),
			ResultPartitionType.BLOCKING_PERSISTENT);

	// remove this node so the OutputFormatVertex will not shown in the final JobGraph.
	vertices.remove(node);
	return true;
}