Java Code Examples for org.apache.flink.runtime.operators.DriverStrategy#SORTED_GROUP_REDUCE

@Override
public DriverStrategy getStrategy() {
	return DriverStrategy.SORTED_GROUP_REDUCE;
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		if(in.getSource().getOptimizerNode() instanceof PartitionNode) {
			LOG.warn("Cannot automatically inject combiner for GroupReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
		}
		// adjust a sort (changes grouping, so it must be for this driver to combining sort
		if (in.getLocalStrategy() == LocalStrategy.SORT) {
			if (!in.getLocalStrategyKeys().isValidUnorderedPrefix(this.keys)) {
				throw new RuntimeException("Bug: Inconsistent sort for group strategy.");
			}
			in.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
								in.getLocalStrategySortOrder());
		}
		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", in,
										DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
	} else {
		// non forward case. all local properties are killed anyways, so we can safely plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);

		// create an input node for combine with same parallelism as input node
		GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator()
				.getName()+")", toCombiner, DriverStrategy.SORTED_GROUP_COMBINE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		// set sorting comparator key info
		combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
		// set grouping comparator key info
		combiner.setDriverKeyInfo(this.keyList, 1);
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
								in.getShipStrategySortOrder(), in.getDataExchangeMode());
		if (in.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
			toReducer.setDataDistribution(in.getDataDistribution());
		}
		toReducer.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
									in.getLocalStrategySortOrder());

		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
	}
}
 

@Override
public DriverStrategy getStrategy() {
	return DriverStrategy.SORTED_GROUP_REDUCE;
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
}
 

@Override
public DriverStrategy getStrategy() {
	return DriverStrategy.SORTED_GROUP_REDUCE;
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		if(in.getSource().getOptimizerNode() instanceof PartitionNode) {
			LOG.warn("Cannot automatically inject combiner for GroupReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
		}
		// adjust a sort (changes grouping, so it must be for this driver to combining sort
		if (in.getLocalStrategy() == LocalStrategy.SORT) {
			if (!in.getLocalStrategyKeys().isValidUnorderedPrefix(this.keys)) {
				throw new RuntimeException("Bug: Inconsistent sort for group strategy.");
			}
			in.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
								in.getLocalStrategySortOrder());
		}
		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", in,
										DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
	} else {
		// non forward case. all local properties are killed anyways, so we can safely plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);

		// create an input node for combine with same parallelism as input node
		GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator()
				.getName()+")", toCombiner, DriverStrategy.SORTED_GROUP_COMBINE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		// set sorting comparator key info
		combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
		// set grouping comparator key info
		combiner.setDriverKeyInfo(this.keyList, 1);
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
								in.getShipStrategySortOrder(), in.getDataExchangeMode());
		if (in.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
			toReducer.setDataDistribution(in.getDataDistribution());
		}
		toReducer.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
									in.getLocalStrategySortOrder());

		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
	}
}
 

@Override
public DriverStrategy getStrategy() {
	return DriverStrategy.SORTED_GROUP_REDUCE;
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
}
 

@Override
public DriverStrategy getStrategy() {
	return DriverStrategy.SORTED_GROUP_REDUCE;
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		if(in.getSource().getOptimizerNode() instanceof PartitionNode) {
			LOG.warn("Cannot automatically inject combiner for GroupReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
		}
		// adjust a sort (changes grouping, so it must be for this driver to combining sort
		if (in.getLocalStrategy() == LocalStrategy.SORT) {
			if (!in.getLocalStrategyKeys().isValidUnorderedPrefix(this.keys)) {
				throw new RuntimeException("Bug: Inconsistent sort for group strategy.");
			}
			in.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
								in.getLocalStrategySortOrder());
		}
		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", in,
										DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
	} else {
		// non forward case. all local properties are killed anyways, so we can safely plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);

		// create an input node for combine with same parallelism as input node
		GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator()
				.getName()+")", toCombiner, DriverStrategy.SORTED_GROUP_COMBINE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		// set sorting comparator key info
		combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
		// set grouping comparator key info
		combiner.setDriverKeyInfo(this.keyList, 1);
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
								in.getShipStrategySortOrder(), in.getDataExchangeMode());
		if (in.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
			toReducer.setDataDistribution(in.getDataDistribution());
		}
		toReducer.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
									in.getLocalStrategySortOrder());

		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
	}
}
 

@Override
public DriverStrategy getStrategy() {
	return DriverStrategy.SORTED_GROUP_REDUCE;
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
}