org.apache.flink.optimizer.operators.OperatorDescriptorSingle Java Examples

protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
		List<PlanNode> target, CostEstimator estimator)
{
	for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
		final Channel in = template.clone();
		ilp.parameterizeChannel(in);
		
		// instantiate a candidate, if the instantiated local properties meet one possible local property set
		outer:
		for (OperatorDescriptorSingle dps: getPossibleProperties()) {
			for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
				if (ilps.isMetBy(in.getLocalProperties())) {
					in.setRequiredLocalProps(ilps);
					instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
					break outer;
				}
			}
		}
	}
}
 

protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
		List<PlanNode> target, CostEstimator estimator)
{
	for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
		final Channel in = template.clone();
		ilp.parameterizeChannel(in);
		
		// instantiate a candidate, if the instantiated local properties meet one possible local property set
		outer:
		for (OperatorDescriptorSingle dps: getPossibleProperties()) {
			for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
				if (ilps.isMetBy(in.getLocalProperties())) {
					in.setRequiredLocalProps(ilps);
					instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
					break outer;
				}
			}
		}
	}
}
 

protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
		List<PlanNode> target, CostEstimator estimator)
{
	for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
		final Channel in = template.clone();
		ilp.parameterizeChannel(in);
		
		// instantiate a candidate, if the instantiated local properties meet one possible local property set
		outer:
		for (OperatorDescriptorSingle dps: getPossibleProperties()) {
			for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
				if (ilps.isMetBy(in.getLocalProperties())) {
					in.setRequiredLocalProps(ilps);
					instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
					break outer;
				}
			}
		}
	}
}
 

private List<OperatorDescriptorSingle> initPossibleProperties() {

		// check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
		Ordering groupOrder = getOperator().getGroupOrder();
		if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
			groupOrder = null;
		}

		OperatorDescriptorSingle props = (this.keys == null ?
				new AllGroupCombineProperties() :
				new GroupCombineProperties(this.keys, groupOrder));

		return Collections.singletonList(props);
	}
 

public PartitionNode(PartitionOperatorBase<?> operator) {
	super(operator);
	
	OperatorDescriptorSingle descr = new PartitionDescriptor(
				this.getOperator().getPartitionMethod(), this.keys, operator.getOrdering(), operator.getCustomPartitioner(),
				operator.getDistribution());
	this.possibleProperties = Collections.singletonList(descr);
}
 

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
	// get what we inherit and what is preserved by our user code 
	final InterestingProperties props = getInterestingProperties().filterByCodeAnnotations(this, 0);
	
	// add all properties relevant to this node
	for (OperatorDescriptorSingle dps : getPossibleProperties()) {
		for (RequestedGlobalProperties gp : dps.getPossibleGlobalProperties()) {
			
			if (gp.getPartitioning().isPartitionedOnKey()) {
				// make sure that among the same partitioning types, we do not push anything down that has fewer key fields
				
				for (RequestedGlobalProperties contained : props.getGlobalProperties()) {
					if (contained.getPartitioning() == gp.getPartitioning() && gp.getPartitionedFields().isValidSubset(contained.getPartitionedFields())) {
						props.getGlobalProperties().remove(contained);
						break;
					}
				}
			}
			
			props.addGlobalProperties(gp);
		}
		
		for (RequestedLocalProperties lp : dps.getPossibleLocalProperties()) {
			props.addLocalProperties(lp);
		}
	}
	this.inConn.setInterestingProperties(props);
	
	for (DagConnection conn : getBroadcastConnections()) {
		conn.setInterestingProperties(new InterestingProperties());
	}
}
 

private List<OperatorDescriptorSingle> initPossibleProperties() {

		// check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
		Ordering groupOrder = getOperator().getGroupOrder();
		if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
			groupOrder = null;
		}

		OperatorDescriptorSingle props = (this.keys == null ?
				new AllGroupCombineProperties() :
				new GroupCombineProperties(this.keys, groupOrder));

		return Collections.singletonList(props);
	}
 

private List<OperatorDescriptorSingle> initPossibleProperties(Partitioner<?> customPartitioner) {
	// see if an internal hint dictates the strategy to use
	final Configuration conf = getOperator().getParameters();
	final String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);

	final boolean useCombiner;
	if (localStrategy != null) {
		if (Optimizer.HINT_LOCAL_STRATEGY_SORT.equals(localStrategy)) {
			useCombiner = false;
		}
		else if (Optimizer.HINT_LOCAL_STRATEGY_COMBINING_SORT.equals(localStrategy)) {
			if (!isCombineable()) {
				Optimizer.LOG.warn("Strategy hint for GroupReduce '" + getOperator().getName() +
					"' requires combinable reduce, but user function is not marked combinable.");
			}
			useCombiner = true;
		} else {
			throw new CompilerException("Invalid local strategy hint for match contract: " + localStrategy);
		}
	} else {
		useCombiner = isCombineable();
	}
	
	// check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
	Ordering groupOrder = null;
	if (getOperator() != null) {
		groupOrder = getOperator().getGroupOrder();
		if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
			groupOrder = null;
		}
	}
	
	OperatorDescriptorSingle props = useCombiner ?
		(this.keys == null ? new AllGroupWithPartialPreGroupProperties() : new GroupReduceWithCombineProperties(this.keys, groupOrder, customPartitioner)) :
		(this.keys == null ? new AllGroupReduceProperties() : new GroupReduceProperties(this.keys, groupOrder, customPartitioner));

	return Collections.singletonList(props);
}
 

private List<OperatorDescriptorSingle> initPossibleProperties(Partitioner<?> customPartitioner) {
	// see if an internal hint dictates the strategy to use
	final Configuration conf = getOperator().getParameters();
	final String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);

	final boolean useCombiner;
	if (localStrategy != null) {
		if (Optimizer.HINT_LOCAL_STRATEGY_SORT.equals(localStrategy)) {
			useCombiner = false;
		}
		else if (Optimizer.HINT_LOCAL_STRATEGY_COMBINING_SORT.equals(localStrategy)) {
			if (!isCombineable()) {
				Optimizer.LOG.warn("Strategy hint for GroupReduce '" + getOperator().getName() +
					"' requires combinable reduce, but user function is not marked combinable.");
			}
			useCombiner = true;
		} else {
			throw new CompilerException("Invalid local strategy hint for match contract: " + localStrategy);
		}
	} else {
		useCombiner = isCombineable();
	}
	
	// check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
	Ordering groupOrder = null;
	if (getOperator() != null) {
		groupOrder = getOperator().getGroupOrder();
		if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
			groupOrder = null;
		}
	}
	
	OperatorDescriptorSingle props = useCombiner ?
		(this.keys == null ? new AllGroupWithPartialPreGroupProperties() : new GroupReduceWithCombineProperties(this.keys, groupOrder, customPartitioner)) :
		(this.keys == null ? new AllGroupReduceProperties() : new GroupReduceProperties(this.keys, groupOrder, customPartitioner));

	return Collections.singletonList(props);
}
 

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
	// get what we inherit and what is preserved by our user code 
	final InterestingProperties props = getInterestingProperties().filterByCodeAnnotations(this, 0);
	
	// add all properties relevant to this node
	for (OperatorDescriptorSingle dps : getPossibleProperties()) {
		for (RequestedGlobalProperties gp : dps.getPossibleGlobalProperties()) {
			
			if (gp.getPartitioning().isPartitionedOnKey()) {
				// make sure that among the same partitioning types, we do not push anything down that has fewer key fields
				
				for (RequestedGlobalProperties contained : props.getGlobalProperties()) {
					if (contained.getPartitioning() == gp.getPartitioning() && gp.getPartitionedFields().isValidSubset(contained.getPartitionedFields())) {
						props.getGlobalProperties().remove(contained);
						break;
					}
				}
			}
			
			props.addGlobalProperties(gp);
		}
		
		for (RequestedLocalProperties lp : dps.getPossibleLocalProperties()) {
			props.addLocalProperties(lp);
		}
	}
	this.inConn.setInterestingProperties(props);
	
	for (DagConnection conn : getBroadcastConnections()) {
		conn.setInterestingProperties(new InterestingProperties());
	}
}
 

public PartitionNode(PartitionOperatorBase<?> operator) {
	super(operator);
	
	OperatorDescriptorSingle descr = new PartitionDescriptor(
				this.getOperator().getPartitionMethod(), this.keys, operator.getOrdering(), operator.getCustomPartitioner(),
				operator.getDistribution());
	this.possibleProperties = Collections.singletonList(descr);
}
 

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
	// get what we inherit and what is preserved by our user code 
	final InterestingProperties props = getInterestingProperties().filterByCodeAnnotations(this, 0);
	
	// add all properties relevant to this node
	for (OperatorDescriptorSingle dps : getPossibleProperties()) {
		for (RequestedGlobalProperties gp : dps.getPossibleGlobalProperties()) {
			
			if (gp.getPartitioning().isPartitionedOnKey()) {
				// make sure that among the same partitioning types, we do not push anything down that has fewer key fields
				
				for (RequestedGlobalProperties contained : props.getGlobalProperties()) {
					if (contained.getPartitioning() == gp.getPartitioning() && gp.getPartitionedFields().isValidSubset(contained.getPartitionedFields())) {
						props.getGlobalProperties().remove(contained);
						break;
					}
				}
			}
			
			props.addGlobalProperties(gp);
		}
		
		for (RequestedLocalProperties lp : dps.getPossibleLocalProperties()) {
			props.addLocalProperties(lp);
		}
	}
	this.inConn.setInterestingProperties(props);
	
	for (DagConnection conn : getBroadcastConnections()) {
		conn.setInterestingProperties(new InterestingProperties());
	}
}
 

public PartitionNode(PartitionOperatorBase<?> operator) {
	super(operator);
	
	OperatorDescriptorSingle descr = new PartitionDescriptor(
				this.getOperator().getPartitionMethod(), this.keys, operator.getOrdering(), operator.getCustomPartitioner(),
				operator.getDistribution());
	this.possibleProperties = Collections.singletonList(descr);
}
 

private List<OperatorDescriptorSingle> initPossibleProperties() {

		// check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
		Ordering groupOrder = getOperator().getGroupOrder();
		if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
			groupOrder = null;
		}

		OperatorDescriptorSingle props = (this.keys == null ?
				new AllGroupCombineProperties() :
				new GroupCombineProperties(this.keys, groupOrder));

		return Collections.singletonList(props);
	}
 

private List<OperatorDescriptorSingle> initPossibleProperties(Partitioner<?> customPartitioner) {
	// see if an internal hint dictates the strategy to use
	final Configuration conf = getOperator().getParameters();
	final String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);

	final boolean useCombiner;
	if (localStrategy != null) {
		if (Optimizer.HINT_LOCAL_STRATEGY_SORT.equals(localStrategy)) {
			useCombiner = false;
		}
		else if (Optimizer.HINT_LOCAL_STRATEGY_COMBINING_SORT.equals(localStrategy)) {
			if (!isCombineable()) {
				Optimizer.LOG.warn("Strategy hint for GroupReduce '" + getOperator().getName() +
					"' requires combinable reduce, but user function is not marked combinable.");
			}
			useCombiner = true;
		} else {
			throw new CompilerException("Invalid local strategy hint for match contract: " + localStrategy);
		}
	} else {
		useCombiner = isCombineable();
	}
	
	// check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
	Ordering groupOrder = null;
	if (getOperator() != null) {
		groupOrder = getOperator().getGroupOrder();
		if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
			groupOrder = null;
		}
	}
	
	OperatorDescriptorSingle props = useCombiner ?
		(this.keys == null ? new AllGroupWithPartialPreGroupProperties() : new GroupReduceWithCombineProperties(this.keys, groupOrder, customPartitioner)) :
		(this.keys == null ? new AllGroupReduceProperties() : new GroupReduceProperties(this.keys, groupOrder, customPartitioner));

	return Collections.singletonList(props);
}
 

public UnaryOperatorNode(String name, FieldSet keys, List<OperatorDescriptorSingle> operators) {
	super(keys);
	
	this.operators = operators;
	this.name = name;
}
 

@Override
protected List<OperatorDescriptorSingle> getPossibleProperties() {
	return this.operators;
}
 

public SortPartitionNode(SortPartitionOperatorBase<?> operator) {
	super(operator);
	
	OperatorDescriptorSingle descr = new SortPartitionDescriptor(operator.getPartitionOrdering());
	this.possibleProperties = Collections.singletonList(descr);
}
 

@Override
protected List<OperatorDescriptorSingle> getPossibleProperties() {
	return this.possibleProperties;
}
 

@Override
protected List<OperatorDescriptorSingle> getPossibleProperties() {
	return this.possibleProperties;
}
 

public UnaryOperatorNode(String name, FieldSet keys, List<OperatorDescriptorSingle> operators) {
	super(keys);
	
	this.operators = operators;
	this.name = name;
}
 

public FilterNode(FilterOperatorBase<?, ?> operator) {
	super(operator);
	this.possibleProperties = Collections.<OperatorDescriptorSingle>singletonList(new FilterDescriptor());
}
 

@Override
protected List<OperatorDescriptorSingle> getPossibleProperties() {
	return this.possibleProperties;
}
 

@Override
protected List<OperatorDescriptorSingle> getPossibleProperties() {
	return this.possibleProperties;
}
 

/**
 * Creates a new MapNode for the given contract.
 * 
 * @param operator The map partition contract object.
 */
public MapPartitionNode(SingleInputOperator<?, ?, ?> operator) {
	super(operator);
	
	this.possibleProperties = Collections.<OperatorDescriptorSingle>singletonList(new MapPartitionDescriptor());
}
 

protected List<OperatorDescriptorSingle> getPossibleProperties() {
	return Collections.<OperatorDescriptorSingle>singletonList(new NoOpDescriptor());
}
 

@Override
protected List<OperatorDescriptorSingle> getPossibleProperties() {
	return this.possibleProperties;
}
 

public SortPartitionNode(SortPartitionOperatorBase<?> operator) {
	super(operator);
	
	OperatorDescriptorSingle descr = new SortPartitionDescriptor(operator.getPartitionOrdering());
	this.possibleProperties = Collections.singletonList(descr);
}
 

@Override
protected List<OperatorDescriptorSingle> getPossibleProperties() {
	return this.operators;
}
 

@Override
protected List<OperatorDescriptorSingle> getPossibleProperties() {
	return this.possibleProperties;
}