org.apache.flink.optimizer.dataproperties.RequestedLocalProperties Java Examples

@Override
public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
		LocalProperties produced1, LocalProperties produced2) {
	int numRelevantFields = this.keys1.size();

	Ordering prod1 = produced1.getOrdering();
	Ordering prod2 = produced2.getOrdering();

	if (prod1 == null || prod2 == null || prod1.getNumberOfFields() < numRelevantFields
			|| prod2.getNumberOfFields() < numRelevantFields) {
		throw new CompilerException("The given properties do not meet this operators requirements.");
	}

	for (int i = 0; i < numRelevantFields; i++) {
		if (prod1.getOrder(i) != prod2.getOrder(i)) {
			return false;
		}
	}
	return true;
}
 

@Override
public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
		LocalProperties produced1, LocalProperties produced2) {
	int numRelevantFields = this.keys1.size();

	Ordering prod1 = produced1.getOrdering();
	Ordering prod2 = produced2.getOrdering();

	if (prod1 == null || prod2 == null || prod1.getNumberOfFields() < numRelevantFields
			|| prod2.getNumberOfFields() < numRelevantFields) {
		throw new CompilerException("The given properties do not meet this operators requirements.");
	}

	for (int i = 0; i < numRelevantFields; i++) {
		if (prod1.getOrder(i) != prod2.getOrder(i)) {
			return false;
		}
	}
	return true;
}
 

protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
		List<PlanNode> target, CostEstimator estimator)
{
	for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
		final Channel in = template.clone();
		ilp.parameterizeChannel(in);
		
		// instantiate a candidate, if the instantiated local properties meet one possible local property set
		outer:
		for (OperatorDescriptorSingle dps: getPossibleProperties()) {
			for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
				if (ilps.isMetBy(in.getLocalProperties())) {
					in.setRequiredLocalProps(ilps);
					instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
					break outer;
				}
			}
		}
	}
}
 

protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
		List<PlanNode> target, CostEstimator estimator)
{
	for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
		final Channel in = template.clone();
		ilp.parameterizeChannel(in);
		
		// instantiate a candidate, if the instantiated local properties meet one possible local property set
		outer:
		for (OperatorDescriptorSingle dps: getPossibleProperties()) {
			for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
				if (ilps.isMetBy(in.getLocalProperties())) {
					in.setRequiredLocalProps(ilps);
					instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
					break outer;
				}
			}
		}
	}
}
 

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
	final InterestingProperties iProps = new InterestingProperties();

	{
		final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties();
		iProps.addGlobalProperties(partitioningProps);
	}

	{
		final Ordering localOrder = getOperator().getLocalOrder();
		final RequestedLocalProperties orderProps = new RequestedLocalProperties();
		if (localOrder != null) {
			orderProps.setOrdering(localOrder);
		}
		iProps.addLocalProperties(orderProps);
	}
	
	this.input.setInterestingProperties(iProps);
}
 

protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
		List<PlanNode> target, CostEstimator estimator)
{
	for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
		final Channel in = template.clone();
		ilp.parameterizeChannel(in);
		
		// instantiate a candidate, if the instantiated local properties meet one possible local property set
		outer:
		for (OperatorDescriptorSingle dps: getPossibleProperties()) {
			for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
				if (ilps.isMetBy(in.getLocalProperties())) {
					in.setRequiredLocalProps(ilps);
					instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
					break outer;
				}
			}
		}
	}
}
 

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
	final InterestingProperties iProps = new InterestingProperties();

	{
		final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties();
		iProps.addGlobalProperties(partitioningProps);
	}

	{
		final Ordering localOrder = getOperator().getLocalOrder();
		final RequestedLocalProperties orderProps = new RequestedLocalProperties();
		if (localOrder != null) {
			orderProps.setOrdering(localOrder);
		}
		iProps.addLocalProperties(orderProps);
	}
	
	this.input.setInterestingProperties(iProps);
}
 

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
	// get what we inherit and what is preserved by our user code 
	final InterestingProperties props = getInterestingProperties().filterByCodeAnnotations(this, 0);
	
	// add all properties relevant to this node
	for (OperatorDescriptorSingle dps : getPossibleProperties()) {
		for (RequestedGlobalProperties gp : dps.getPossibleGlobalProperties()) {
			
			if (gp.getPartitioning().isPartitionedOnKey()) {
				// make sure that among the same partitioning types, we do not push anything down that has fewer key fields
				
				for (RequestedGlobalProperties contained : props.getGlobalProperties()) {
					if (contained.getPartitioning() == gp.getPartitioning() && gp.getPartitionedFields().isValidSubset(contained.getPartitionedFields())) {
						props.getGlobalProperties().remove(contained);
						break;
					}
				}
			}
			
			props.addGlobalProperties(gp);
		}
		
		for (RequestedLocalProperties lp : dps.getPossibleLocalProperties()) {
			props.addLocalProperties(lp);
		}
	}
	this.inConn.setInterestingProperties(props);
	
	for (DagConnection conn : getBroadcastConnections()) {
		conn.setInterestingProperties(new InterestingProperties());
	}
}
 

@Override
protected List<RequestedLocalProperties> createPossibleLocalProperties() {
	// set partition order as required local property
	RequestedLocalProperties rlp = new RequestedLocalProperties();
	rlp.setOrdering(this.partitionOrder);

	return Collections.singletonList(rlp);
}
 

@Override
protected List<RequestedLocalProperties> createPossibleLocalProperties() {
	RequestedLocalProperties props = new RequestedLocalProperties();
	if (this.ordering == null) {
		props.setGroupedFields(this.keys);
	} else {
		props.setOrdering(this.ordering);
	}
	return Collections.singletonList(props);
}
 

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) { 
	final InterestingProperties props = getInterestingProperties();
	
	// if no other properties exist, add the pruned trivials back
	if (props.getGlobalProperties().isEmpty()) {
		props.addGlobalProperties(new RequestedGlobalProperties());
	}
	props.addLocalProperties(new RequestedLocalProperties());
	this.input1.setInterestingProperties(props.clone());
	this.input2.setInterestingProperties(props.clone());
	
	this.channelProps = props.getGlobalProperties();
}
 

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) { 
	final InterestingProperties props = getInterestingProperties();
	
	// if no other properties exist, add the pruned trivials back
	if (props.getGlobalProperties().isEmpty()) {
		props.addGlobalProperties(new RequestedGlobalProperties());
	}
	props.addLocalProperties(new RequestedLocalProperties());
	this.input1.setInterestingProperties(props.clone());
	this.input2.setInterestingProperties(props.clone());
	
	this.channelProps = props.getGlobalProperties();
}
 

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
	// get what we inherit and what is preserved by our user code 
	final InterestingProperties props = getInterestingProperties().filterByCodeAnnotations(this, 0);
	
	// add all properties relevant to this node
	for (OperatorDescriptorSingle dps : getPossibleProperties()) {
		for (RequestedGlobalProperties gp : dps.getPossibleGlobalProperties()) {
			
			if (gp.getPartitioning().isPartitionedOnKey()) {
				// make sure that among the same partitioning types, we do not push anything down that has fewer key fields
				
				for (RequestedGlobalProperties contained : props.getGlobalProperties()) {
					if (contained.getPartitioning() == gp.getPartitioning() && gp.getPartitionedFields().isValidSubset(contained.getPartitionedFields())) {
						props.getGlobalProperties().remove(contained);
						break;
					}
				}
			}
			
			props.addGlobalProperties(gp);
		}
		
		for (RequestedLocalProperties lp : dps.getPossibleLocalProperties()) {
			props.addLocalProperties(lp);
		}
	}
	this.inConn.setInterestingProperties(props);
	
	for (DagConnection conn : getBroadcastConnections()) {
		conn.setInterestingProperties(new InterestingProperties());
	}
}
 

@Override
protected List<RequestedLocalProperties> createPossibleLocalProperties() {
	// set partition order as required local property
	RequestedLocalProperties rlp = new RequestedLocalProperties();
	rlp.setOrdering(this.partitionOrder);

	return Collections.singletonList(rlp);
}
 

@Override
protected List<RequestedLocalProperties> createPossibleLocalProperties() {
	RequestedLocalProperties props = new RequestedLocalProperties();
	if (this.ordering == null) {
		props.setGroupedFields(this.keys);
	} else {
		props.setOrdering(this.ordering);
	}
	return Collections.singletonList(props);
}
 

@Override
protected List<OperatorDescriptorDual.LocalPropertiesPair> createPossibleLocalProperties() {
	RequestedLocalProperties sort1 = new RequestedLocalProperties(this.ordering1);
	RequestedLocalProperties sort2 = new RequestedLocalProperties(this.ordering2);
	return Collections.singletonList(new OperatorDescriptorDual.LocalPropertiesPair(sort1, sort2));
}
 

@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
	// check if we have a cached version
	if (this.cachedPlans != null) {
		return this.cachedPlans;
	}
	
	// calculate alternative sub-plans for predecessor
	List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator);
	List<PlanNode> outputPlans = new ArrayList<PlanNode>();
	
	final int parallelism = getParallelism();
	final int inDop = getPredecessorNode().getParallelism();

	final ExecutionMode executionMode = this.input.getDataExchangeMode();
	final boolean dopChange = parallelism != inDop;
	final boolean breakPipeline = this.input.isBreakingPipeline();

	InterestingProperties ips = this.input.getInterestingProperties();
	for (PlanNode p : subPlans) {
		for (RequestedGlobalProperties gp : ips.getGlobalProperties()) {
			for (RequestedLocalProperties lp : ips.getLocalProperties()) {
				Channel c = new Channel(p);
				gp.parameterizeChannel(c, dopChange, executionMode, breakPipeline);
				lp.parameterizeChannel(c);
				c.setRequiredLocalProps(lp);
				c.setRequiredGlobalProps(gp);
				
				// no need to check whether the created properties meet what we need in case
				// of ordering or global ordering, because the only interesting properties we have
				// are what we require
				outputPlans.add(new SinkPlanNode(this, "DataSink ("+this.getOperator().getName()+")" ,c));
			}
		}
	}
	
	// cost and prune the plans
	for (PlanNode node : outputPlans) {
		estimator.costOperator(node);
	}
	prunePlanAlternatives(outputPlans);

	this.cachedPlans = outputPlans;
	return outputPlans;
}
 

public RequestedLocalProperties getProperties2() {
	return this.props2;
}
 

public void setRequiredLocalProps(RequestedLocalProperties requiredLocalProps) {
	this.requiredLocalProps = requiredLocalProps;
}
 

@Override
protected List<RequestedLocalProperties> createPossibleLocalProperties() {
	return Collections.singletonList(new RequestedLocalProperties());
}
 

@Override
protected List<RequestedLocalProperties> createPossibleLocalProperties() {
	RequestedLocalProperties props = new RequestedLocalProperties();
	props.setGroupedFields(this.keys);
	return Collections.singletonList(props);
}
 

@Override
public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
	LocalProperties produced1, LocalProperties produced2) {
	return true;
}
 

@Override
protected List<RequestedLocalProperties> createPossibleLocalProperties() {
	return Collections.singletonList(new RequestedLocalProperties());
}
 

@Override
public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
		LocalProperties produced1, LocalProperties produced2) {
	return true;
}
 

public List<RequestedLocalProperties> getPossibleLocalProperties() {
	if (this.localProps == null) {
		this.localProps = createPossibleLocalProperties();
	}
	return this.localProps;
}
 

@Override
public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
		LocalProperties produced1, LocalProperties produced2) {
	return true;
}
 

@Override
public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
		LocalProperties produced1, LocalProperties produced2) {
	return true;
}
 

@Override
protected List<LocalPropertiesPair> createPossibleLocalProperties() {
	RequestedLocalProperties sort = new RequestedLocalProperties(Utils.createOrdering(this.keys1));
	RequestedLocalProperties none = new RequestedLocalProperties();
	return Collections.singletonList(new LocalPropertiesPair(sort, none));
}
 

public abstract boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
LocalProperties produced1, LocalProperties produced2);
 

@Override
protected List<RequestedLocalProperties> createPossibleLocalProperties() {
	return Collections.singletonList(new RequestedLocalProperties());
}