Java Code Examples for org.apache.flink.api.common.operators.Ordering#getNumberOfFields()

private List<OperatorDescriptorDual> initializeDataProperties() {
	Ordering groupOrder1 = null;
	Ordering groupOrder2 = null;

	CoGroupRawOperatorBase<?, ?, ?, ?> cgc = getOperator();
	groupOrder1 = cgc.getGroupOrderForInputOne();
	groupOrder2 = cgc.getGroupOrderForInputTwo();

	if (groupOrder1 != null && groupOrder1.getNumberOfFields() == 0) {
		groupOrder1 = null;
	}
	if (groupOrder2 != null && groupOrder2.getNumberOfFields() == 0) {
		groupOrder2 = null;
	}

	return Collections.<OperatorDescriptorDual>singletonList(new CoGroupRawDescriptor(this.keys1, this.keys2, groupOrder1, groupOrder2));
}
 

public GroupReduceProperties(FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) {
	super(groupKeys);
	
	// if we have an additional ordering, construct the ordering to have primarily the grouping fields
	if (additionalOrderKeys != null) {
		this.ordering = new Ordering();
		for (Integer key : this.keyList) {
			this.ordering.appendOrdering(key, null, Order.ANY);
		}
	
		// and next the additional order fields
		for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
			Integer field = additionalOrderKeys.getFieldNumber(i);
			Order order = additionalOrderKeys.getOrder(i);
			this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
		}
	}
	else {
		this.ordering = null;
	}
	
	this.customPartitioner = customPartitioner;
}
 

public GroupReduceWithCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) {
	super(groupKeys);
	
	// if we have an additional ordering, construct the ordering to have primarily the grouping fields
	if (additionalOrderKeys != null) {
		this.ordering = new Ordering();
		for (Integer key : this.keyList) {
			this.ordering.appendOrdering(key, null, Order.ANY);
		}
	
		// and next the additional order fields
		for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
			Integer field = additionalOrderKeys.getFieldNumber(i);
			Order order = additionalOrderKeys.getOrder(i);
			this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
		}
	} else {
		this.ordering = null;
	}
	
	this.customPartitioner = customPartitioner;
}
 

private List<OperatorDescriptorDual> initializeDataProperties(Partitioner<?> customPartitioner) {
	Ordering groupOrder1 = null;
	Ordering groupOrder2 = null;
	
	CoGroupOperatorBase<?, ?, ?, ?> cgc = getOperator();
	groupOrder1 = cgc.getGroupOrderForInputOne();
	groupOrder2 = cgc.getGroupOrderForInputTwo();
		
	if (groupOrder1 != null && groupOrder1.getNumberOfFields() == 0) {
		groupOrder1 = null;
	}
	if (groupOrder2 != null && groupOrder2.getNumberOfFields() == 0) {
		groupOrder2 = null;
	}
	
	CoGroupDescriptor descr = new CoGroupDescriptor(this.keys1, this.keys2, groupOrder1, groupOrder2);
	if (customPartitioner != null) {
		descr.setCustomPartitioner(customPartitioner);
	}
	
	return Collections.<OperatorDescriptorDual>singletonList(descr);
}
 

public GroupCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys) {
	super(groupKeys);

	// if we have an additional ordering, construct the ordering to have primarily the grouping fields
	
	this.ordering = new Ordering();
	for (Integer key : this.keyList) {
		this.ordering.appendOrdering(key, null, Order.ANY);
	}

	// and next the additional order fields
	if (additionalOrderKeys != null) {
		for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
			Integer field = additionalOrderKeys.getFieldNumber(i);
			Order order = additionalOrderKeys.getOrder(i);
			this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
		}
	}

}
 

@Override
public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
		LocalProperties produced1, LocalProperties produced2) {
	int numRelevantFields = this.keys1.size();

	Ordering prod1 = produced1.getOrdering();
	Ordering prod2 = produced2.getOrdering();

	if (prod1 == null || prod2 == null || prod1.getNumberOfFields() < numRelevantFields
			|| prod2.getNumberOfFields() < numRelevantFields) {
		throw new CompilerException("The given properties do not meet this operators requirements.");
	}

	for (int i = 0; i < numRelevantFields; i++) {
		if (prod1.getOrder(i) != prod2.getOrder(i)) {
			return false;
		}
	}
	return true;
}
 

public GroupReduceWithCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) {
	super(groupKeys);
	
	// if we have an additional ordering, construct the ordering to have primarily the grouping fields
	if (additionalOrderKeys != null) {
		this.ordering = new Ordering();
		for (Integer key : this.keyList) {
			this.ordering.appendOrdering(key, null, Order.ANY);
		}
	
		// and next the additional order fields
		for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
			Integer field = additionalOrderKeys.getFieldNumber(i);
			Order order = additionalOrderKeys.getOrder(i);
			this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
		}
	} else {
		this.ordering = null;
	}
	
	this.customPartitioner = customPartitioner;
}
 

public GroupReduceWithCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) {
	super(groupKeys);
	
	// if we have an additional ordering, construct the ordering to have primarily the grouping fields
	if (additionalOrderKeys != null) {
		this.ordering = new Ordering();
		for (Integer key : this.keyList) {
			this.ordering.appendOrdering(key, null, Order.ANY);
		}
	
		// and next the additional order fields
		for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
			Integer field = additionalOrderKeys.getFieldNumber(i);
			Order order = additionalOrderKeys.getOrder(i);
			this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
		}
	} else {
		this.ordering = null;
	}
	
	this.customPartitioner = customPartitioner;
}
 

private List<OperatorDescriptorDual> initializeDataProperties(Partitioner<?> customPartitioner) {
	Ordering groupOrder1 = null;
	Ordering groupOrder2 = null;
	
	CoGroupOperatorBase<?, ?, ?, ?> cgc = getOperator();
	groupOrder1 = cgc.getGroupOrderForInputOne();
	groupOrder2 = cgc.getGroupOrderForInputTwo();
		
	if (groupOrder1 != null && groupOrder1.getNumberOfFields() == 0) {
		groupOrder1 = null;
	}
	if (groupOrder2 != null && groupOrder2.getNumberOfFields() == 0) {
		groupOrder2 = null;
	}
	
	CoGroupDescriptor descr = new CoGroupDescriptor(this.keys1, this.keys2, groupOrder1, groupOrder2);
	if (customPartitioner != null) {
		descr.setCustomPartitioner(customPartitioner);
	}
	
	return Collections.<OperatorDescriptorDual>singletonList(descr);
}
 

private List<OperatorDescriptorDual> initializeDataProperties(Partitioner<?> customPartitioner) {
	Ordering groupOrder1 = null;
	Ordering groupOrder2 = null;
	
	CoGroupOperatorBase<?, ?, ?, ?> cgc = getOperator();
	groupOrder1 = cgc.getGroupOrderForInputOne();
	groupOrder2 = cgc.getGroupOrderForInputTwo();
		
	if (groupOrder1 != null && groupOrder1.getNumberOfFields() == 0) {
		groupOrder1 = null;
	}
	if (groupOrder2 != null && groupOrder2.getNumberOfFields() == 0) {
		groupOrder2 = null;
	}
	
	CoGroupDescriptor descr = new CoGroupDescriptor(this.keys1, this.keys2, groupOrder1, groupOrder2);
	if (customPartitioner != null) {
		descr.setCustomPartitioner(customPartitioner);
	}
	
	return Collections.<OperatorDescriptorDual>singletonList(descr);
}
 

private List<OperatorDescriptorDual> initializeDataProperties() {
	Ordering groupOrder1 = null;
	Ordering groupOrder2 = null;

	CoGroupRawOperatorBase<?, ?, ?, ?> cgc = getOperator();
	groupOrder1 = cgc.getGroupOrderForInputOne();
	groupOrder2 = cgc.getGroupOrderForInputTwo();

	if (groupOrder1 != null && groupOrder1.getNumberOfFields() == 0) {
		groupOrder1 = null;
	}
	if (groupOrder2 != null && groupOrder2.getNumberOfFields() == 0) {
		groupOrder2 = null;
	}

	return Collections.<OperatorDescriptorDual>singletonList(new CoGroupRawDescriptor(this.keys1, this.keys2, groupOrder1, groupOrder2));
}
 

@Override
public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
		LocalProperties produced1, LocalProperties produced2) {
	int numRelevantFields = this.keys1.size();

	Ordering prod1 = produced1.getOrdering();
	Ordering prod2 = produced2.getOrdering();

	if (prod1 == null || prod2 == null || prod1.getNumberOfFields() < numRelevantFields
			|| prod2.getNumberOfFields() < numRelevantFields) {
		throw new CompilerException("The given properties do not meet this operators requirements.");
	}

	for (int i = 0; i < numRelevantFields; i++) {
		if (prod1.getOrder(i) != prod2.getOrder(i)) {
			return false;
		}
	}
	return true;
}
 

@Override
public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
		LocalProperties produced1, LocalProperties produced2) {
	int numRelevantFields = this.keys1.size();

	Ordering prod1 = produced1.getOrdering();
	Ordering prod2 = produced2.getOrdering();

	if (prod1 == null || prod2 == null || prod1.getNumberOfFields() < numRelevantFields
			|| prod2.getNumberOfFields() < numRelevantFields) {
		throw new CompilerException("The given properties do not meet this operators requirements.");
	}

	for (int i = 0; i < numRelevantFields; i++) {
		if (prod1.getOrder(i) != prod2.getOrder(i)) {
			return false;
		}
	}
	return true;
}
 

public GroupReduceProperties(FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) {
	super(groupKeys);
	
	// if we have an additional ordering, construct the ordering to have primarily the grouping fields
	if (additionalOrderKeys != null) {
		this.ordering = new Ordering();
		for (Integer key : this.keyList) {
			this.ordering.appendOrdering(key, null, Order.ANY);
		}
	
		// and next the additional order fields
		for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
			Integer field = additionalOrderKeys.getFieldNumber(i);
			Order order = additionalOrderKeys.getOrder(i);
			this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
		}
	}
	else {
		this.ordering = null;
	}
	
	this.customPartitioner = customPartitioner;
}
 

private List<OperatorDescriptorSingle> initPossibleProperties() {

		// check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
		Ordering groupOrder = getOperator().getGroupOrder();
		if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
			groupOrder = null;
		}

		OperatorDescriptorSingle props = (this.keys == null ?
				new AllGroupCombineProperties() :
				new GroupCombineProperties(this.keys, groupOrder));

		return Collections.singletonList(props);
	}
 

public boolean matchesOrderedPartitioning(Ordering o) {
	if (this.partitioning == PartitioningProperty.RANGE_PARTITIONED) {
		if (this.ordering.getNumberOfFields() > o.getNumberOfFields()) {
			return false;
		}
		
		for (int i = 0; i < this.ordering.getNumberOfFields(); i++) {
			if (!this.ordering.getFieldNumber(i).equals(o.getFieldNumber(i))) {
				return false;
			}
			
			// if this one request no order, everything is good
			final Order oo = o.getOrder(i);
			final Order to = this.ordering.getOrder(i);
			if (oo != Order.NONE) {
				if (oo == Order.ANY) {
					// if any order is requested, any not NONE order is good
					if (to == Order.NONE) {
						return false;
					}
				} else if (oo != to) {
					// the orders must be equal
					return false;
				}
			}
		}
		return true;
	} else {
		return false;
	}
}
 

public boolean matchesOrderedPartitioning(Ordering o) {
	if (this.partitioning == PartitioningProperty.RANGE_PARTITIONED) {
		if (this.ordering.getNumberOfFields() > o.getNumberOfFields()) {
			return false;
		}
		
		for (int i = 0; i < this.ordering.getNumberOfFields(); i++) {
			if (!this.ordering.getFieldNumber(i).equals(o.getFieldNumber(i))) {
				return false;
			}
			
			// if this one request no order, everything is good
			final Order oo = o.getOrder(i);
			final Order to = this.ordering.getOrder(i);
			if (oo != Order.NONE) {
				if (oo == Order.ANY) {
					// if any order is requested, any not NONE order is good
					if (to == Order.NONE) {
						return false;
					}
				} else if (oo != to) {
					// the orders must be equal
					return false;
				}
			}
		}
		return true;
	} else {
		return false;
	}
}
 

private List<OperatorDescriptorSingle> initPossibleProperties() {

		// check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
		Ordering groupOrder = getOperator().getGroupOrder();
		if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
			groupOrder = null;
		}

		OperatorDescriptorSingle props = (this.keys == null ?
				new AllGroupCombineProperties() :
				new GroupCombineProperties(this.keys, groupOrder));

		return Collections.singletonList(props);
	}
 

private List<OperatorDescriptorSingle> initPossibleProperties(Partitioner<?> customPartitioner) {
	// see if an internal hint dictates the strategy to use
	final Configuration conf = getOperator().getParameters();
	final String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);

	final boolean useCombiner;
	if (localStrategy != null) {
		if (Optimizer.HINT_LOCAL_STRATEGY_SORT.equals(localStrategy)) {
			useCombiner = false;
		}
		else if (Optimizer.HINT_LOCAL_STRATEGY_COMBINING_SORT.equals(localStrategy)) {
			if (!isCombineable()) {
				Optimizer.LOG.warn("Strategy hint for GroupReduce '" + getOperator().getName() +
					"' requires combinable reduce, but user function is not marked combinable.");
			}
			useCombiner = true;
		} else {
			throw new CompilerException("Invalid local strategy hint for match contract: " + localStrategy);
		}
	} else {
		useCombiner = isCombineable();
	}
	
	// check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
	Ordering groupOrder = null;
	if (getOperator() != null) {
		groupOrder = getOperator().getGroupOrder();
		if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
			groupOrder = null;
		}
	}
	
	OperatorDescriptorSingle props = useCombiner ?
		(this.keys == null ? new AllGroupWithPartialPreGroupProperties() : new GroupReduceWithCombineProperties(this.keys, groupOrder, customPartitioner)) :
		(this.keys == null ? new AllGroupReduceProperties() : new GroupReduceProperties(this.keys, groupOrder, customPartitioner));

	return Collections.singletonList(props);
}
 

private List<OperatorDescriptorSingle> initPossibleProperties() {

		// check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
		Ordering groupOrder = getOperator().getGroupOrder();
		if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
			groupOrder = null;
		}

		OperatorDescriptorSingle props = (this.keys == null ?
				new AllGroupCombineProperties() :
				new GroupCombineProperties(this.keys, groupOrder));

		return Collections.singletonList(props);
	}