org.apache.flink.api.common.operators.DualInputOperator Java Examples
The following examples show how to use
org.apache.flink.api.common.operators.DualInputOperator.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: TwoInputNode.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
/**
* Creates a new two input node for the optimizer plan, representing the given operator.
*
* @param operator The operator that the optimizer DAG node should represent.
*/
public TwoInputNode(DualInputOperator<?, ?, ?, ?> operator) {
super(operator);
int[] k1 = operator.getKeyColumns(0);
int[] k2 = operator.getKeyColumns(1);
this.keys1 = k1 == null || k1.length == 0 ? null : new FieldList(k1);
this.keys2 = k2 == null || k2.length == 0 ? null : new FieldList(k2);
if (this.keys1 != null) {
if (this.keys2 != null) {
if (this.keys1.size() != this.keys2.size()) {
throw new CompilerException("Unequal number of key fields on the two inputs.");
}
} else {
throw new CompilerException("Keys are set on first input, but not on second.");
}
} else if (this.keys2 != null) {
throw new CompilerException("Keys are set on second input, but not on first.");
}
}
Example #2
| Source File: TwoInputNode.java From flink with Apache License 2.0 | 6 votes |
|
/**
* Creates a new two input node for the optimizer plan, representing the given operator.
*
* @param operator The operator that the optimizer DAG node should represent.
*/
public TwoInputNode(DualInputOperator<?, ?, ?, ?> operator) {
super(operator);
int[] k1 = operator.getKeyColumns(0);
int[] k2 = operator.getKeyColumns(1);
this.keys1 = k1 == null || k1.length == 0 ? null : new FieldList(k1);
this.keys2 = k2 == null || k2.length == 0 ? null : new FieldList(k2);
if (this.keys1 != null) {
if (this.keys2 != null) {
if (this.keys1.size() != this.keys2.size()) {
throw new CompilerException("Unequal number of key fields on the two inputs.");
}
} else {
throw new CompilerException("Keys are set on first input, but not on second.");
}
} else if (this.keys2 != null) {
throw new CompilerException("Keys are set on second input, but not on first.");
}
}
Example #3
| Source File: TwoInputNode.java From flink with Apache License 2.0 | 6 votes |
|
/**
* Creates a new two input node for the optimizer plan, representing the given operator.
*
* @param operator The operator that the optimizer DAG node should represent.
*/
public TwoInputNode(DualInputOperator<?, ?, ?, ?> operator) {
super(operator);
int[] k1 = operator.getKeyColumns(0);
int[] k2 = operator.getKeyColumns(1);
this.keys1 = k1 == null || k1.length == 0 ? null : new FieldList(k1);
this.keys2 = k2 == null || k2.length == 0 ? null : new FieldList(k2);
if (this.keys1 != null) {
if (this.keys2 != null) {
if (this.keys1.size() != this.keys2.size()) {
throw new CompilerException("Unequal number of key fields on the two inputs.");
}
} else {
throw new CompilerException("Keys are set on first input, but not on second.");
}
} else if (this.keys2 != null) {
throw new CompilerException("Keys are set on second input, but not on first.");
}
}
Example #4
| Source File: RelationalQueryCompilerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
/**
* Statistics that push towards a repartition merge join. If the join blows the data volume up significantly,
* re-exploiting the sorted order is cheaper.
*/
@Test
public void testQueryWithStatsForRepartitionMerge() {
Plan p = getTPCH3Plan();
p.setExecutionConfig(defaultExecutionConfig);
// set compiler hints
OperatorResolver cr = getContractResolver(p);
DualInputOperator<?, ?, ?, ?> match = cr.getNode(JOIN_NAME);
match.getCompilerHints().setFilterFactor(100f);
testQueryGeneric(100L * 1024 * 1024 * 1024 * 1024, 100L * 1024 * 1024 * 1024 * 1024, 0.01f, 100f, false, true, false, false, true);
}
Example #5
| Source File: RelationalQueryCompilerTest.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Statistics that push towards a repartition merge join. If the join blows the data volume up significantly,
* re-exploiting the sorted order is cheaper.
*/
@Test
public void testQueryWithStatsForRepartitionMerge() {
Plan p = getTPCH3Plan();
p.setExecutionConfig(defaultExecutionConfig);
// set compiler hints
OperatorResolver cr = getContractResolver(p);
DualInputOperator<?, ?, ?, ?> match = cr.getNode(JOIN_NAME);
match.getCompilerHints().setFilterFactor(100f);
testQueryGeneric(100L * 1024 * 1024 * 1024 * 1024, 100L * 1024 * 1024 * 1024 * 1024, 0.01f, 100f, false, true, false, false, true);
}
Example #6
| Source File: RelationalQueryCompilerTest.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Statistics that push towards a repartition merge join. If the join blows the data volume up significantly,
* re-exploiting the sorted order is cheaper.
*/
@Test
public void testQueryWithStatsForRepartitionMerge() throws Exception {
Plan p = getTPCH3Plan();
p.setExecutionConfig(defaultExecutionConfig);
// set compiler hints
OperatorResolver cr = getContractResolver(p);
DualInputOperator<?, ?, ?, ?> match = cr.getNode(JOIN_NAME);
match.getCompilerHints().setFilterFactor(100f);
testQueryGeneric(100L * 1024 * 1024 * 1024 * 1024, 100L * 1024 * 1024 * 1024 * 1024, 0.01f, 100f, false, true, false, false, true);
}
Example #7
| Source File: RelationalQueryCompilerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
private void testQueryGeneric(Plan p, long orderSize, long lineitemSize,
float orderSelectivity, float joinSelectivity,
boolean broadcastOkay, boolean partitionedOkay,
boolean hashJoinFirstOkay, boolean hashJoinSecondOkay, boolean mergeJoinOkay) {
try {
// set statistics
OperatorResolver cr = getContractResolver(p);
GenericDataSourceBase<?, ?> ordersSource = cr.getNode(ORDERS);
GenericDataSourceBase<?, ?> lineItemSource = cr.getNode(LINEITEM);
SingleInputOperator<?, ?, ?> mapper = cr.getNode(MAPPER_NAME);
DualInputOperator<?, ?, ?, ?> joiner = cr.getNode(JOIN_NAME);
setSourceStatistics(ordersSource, orderSize, 100f);
setSourceStatistics(lineItemSource, lineitemSize, 140f);
mapper.getCompilerHints().setAvgOutputRecordSize(16f);
mapper.getCompilerHints().setFilterFactor(orderSelectivity);
joiner.getCompilerHints().setFilterFactor(joinSelectivity);
// compile
final OptimizedPlan plan = compileWithStats(p);
final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);
// get the nodes from the final plan
final SinkPlanNode sink = or.getNode(SINK);
final SingleInputPlanNode reducer = or.getNode(REDUCE_NAME);
final SingleInputPlanNode combiner = reducer.getPredecessor() instanceof SingleInputPlanNode ?
(SingleInputPlanNode) reducer.getPredecessor() : null;
final DualInputPlanNode join = or.getNode(JOIN_NAME);
final SingleInputPlanNode filteringMapper = or.getNode(MAPPER_NAME);
checkStandardStrategies(filteringMapper, join, combiner, reducer, sink);
// check the possible variants and that the variant ia allowed in this specific setting
if (checkBroadcastShipStrategies(join, reducer, combiner)) {
Assert.assertTrue("Broadcast join incorrectly chosen.", broadcastOkay);
if (checkHashJoinStrategies(join, reducer, true)) {
Assert.assertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
} else if (checkHashJoinStrategies(join, reducer, false)) {
Assert.assertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
} else if (checkBroadcastMergeJoin(join, reducer)) {
Assert.assertTrue("Merge join incorrectly chosen", mergeJoinOkay);
} else {
Assert.fail("Plan has no correct hash join or merge join strategies.");
}
}
else if (checkRepartitionShipStrategies(join, reducer, combiner)) {
Assert.assertTrue("Partitioned join incorrectly chosen.", partitionedOkay);
if (checkHashJoinStrategies(join, reducer, true)) {
Assert.assertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
} else if (checkHashJoinStrategies(join, reducer, false)) {
Assert.assertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
} else if (checkRepartitionMergeJoin(join, reducer)) {
Assert.assertTrue("Merge join incorrectly chosen", mergeJoinOkay);
} else {
Assert.fail("Plan has no correct hash join or merge join strategies.");
}
} else {
Assert.fail("Plan has neither correct BC join or partitioned join configuration.");
}
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
Example #8
| Source File: TwoInputNode.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Override
public DualInputOperator<?, ?, ?, ?> getOperator() {
return (DualInputOperator<?, ?, ?, ?>) super.getOperator();
}
Example #9
| Source File: TwoInputNode.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExecutionMode) {
// see if there is a hint that dictates which shipping strategy to use for BOTH inputs
final Configuration conf = getOperator().getParameters();
ShipStrategyType preSet1 = null;
ShipStrategyType preSet2 = null;
String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
if (shipStrategy != null) {
if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.FORWARD;
} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.BROADCAST;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.PARTITION_HASH;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.PARTITION_RANGE;
} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
preSet1 = preSet2 = ShipStrategyType.PARTITION_RANDOM;
} else {
throw new CompilerException("Unknown hint for shipping strategy: " + shipStrategy);
}
}
// see if there is a hint that dictates which shipping strategy to use for the FIRST input
shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, null);
if (shipStrategy != null) {
if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
preSet1 = ShipStrategyType.FORWARD;
} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
preSet1 = ShipStrategyType.BROADCAST;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
preSet1 = ShipStrategyType.PARTITION_HASH;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
preSet1 = ShipStrategyType.PARTITION_RANGE;
} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
preSet1 = ShipStrategyType.PARTITION_RANDOM;
} else {
throw new CompilerException("Unknown hint for shipping strategy of input one: " + shipStrategy);
}
}
// see if there is a hint that dictates which shipping strategy to use for the SECOND input
shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, null);
if (shipStrategy != null) {
if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
preSet2 = ShipStrategyType.FORWARD;
} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
preSet2 = ShipStrategyType.BROADCAST;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
preSet2 = ShipStrategyType.PARTITION_HASH;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
preSet2 = ShipStrategyType.PARTITION_RANGE;
} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
preSet2 = ShipStrategyType.PARTITION_RANDOM;
} else {
throw new CompilerException("Unknown hint for shipping strategy of input two: " + shipStrategy);
}
}
// get the predecessors
DualInputOperator<?, ?, ?, ?> contr = getOperator();
Operator<?> leftPred = contr.getFirstInput();
Operator<?> rightPred = contr.getSecondInput();
OptimizerNode pred1;
DagConnection conn1;
if (leftPred == null) {
throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for first input.");
} else {
pred1 = contractToNode.get(leftPred);
conn1 = new DagConnection(pred1, this, defaultExecutionMode);
if (preSet1 != null) {
conn1.setShipStrategy(preSet1);
}
}
// create the connection and add it
this.input1 = conn1;
pred1.addOutgoingConnection(conn1);
OptimizerNode pred2;
DagConnection conn2;
if (rightPred == null) {
throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for second input.");
} else {
pred2 = contractToNode.get(rightPred);
conn2 = new DagConnection(pred2, this, defaultExecutionMode);
if (preSet2 != null) {
conn2.setShipStrategy(preSet2);
}
}
// create the connection and add it
this.input2 = conn2;
pred2.addOutgoingConnection(conn2);
}
Example #10
| Source File: RelationalQueryCompilerTest.java From flink with Apache License 2.0 | 4 votes |
|
private void testQueryGeneric(Plan p, long orderSize, long lineitemSize,
float orderSelectivity, float joinSelectivity,
boolean broadcastOkay, boolean partitionedOkay,
boolean hashJoinFirstOkay, boolean hashJoinSecondOkay, boolean mergeJoinOkay) {
try {
// set statistics
OperatorResolver cr = getContractResolver(p);
GenericDataSourceBase<?, ?> ordersSource = cr.getNode(ORDERS);
GenericDataSourceBase<?, ?> lineItemSource = cr.getNode(LINEITEM);
SingleInputOperator<?, ?, ?> mapper = cr.getNode(MAPPER_NAME);
DualInputOperator<?, ?, ?, ?> joiner = cr.getNode(JOIN_NAME);
setSourceStatistics(ordersSource, orderSize, 100f);
setSourceStatistics(lineItemSource, lineitemSize, 140f);
mapper.getCompilerHints().setAvgOutputRecordSize(16f);
mapper.getCompilerHints().setFilterFactor(orderSelectivity);
joiner.getCompilerHints().setFilterFactor(joinSelectivity);
// compile
final OptimizedPlan plan = compileWithStats(p);
final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);
// get the nodes from the final plan
final SinkPlanNode sink = or.getNode(SINK);
final SingleInputPlanNode reducer = or.getNode(REDUCE_NAME);
final SingleInputPlanNode combiner = reducer.getPredecessor() instanceof SingleInputPlanNode ?
(SingleInputPlanNode) reducer.getPredecessor() : null;
final DualInputPlanNode join = or.getNode(JOIN_NAME);
final SingleInputPlanNode filteringMapper = or.getNode(MAPPER_NAME);
checkStandardStrategies(filteringMapper, join, combiner, reducer, sink);
// check the possible variants and that the variant ia allowed in this specific setting
if (checkBroadcastShipStrategies(join, reducer, combiner)) {
Assert.assertTrue("Broadcast join incorrectly chosen.", broadcastOkay);
if (checkHashJoinStrategies(join, reducer, true)) {
Assert.assertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
} else if (checkHashJoinStrategies(join, reducer, false)) {
Assert.assertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
} else if (checkBroadcastMergeJoin(join, reducer)) {
Assert.assertTrue("Merge join incorrectly chosen", mergeJoinOkay);
} else {
Assert.fail("Plan has no correct hash join or merge join strategies.");
}
}
else if (checkRepartitionShipStrategies(join, reducer, combiner)) {
Assert.assertTrue("Partitioned join incorrectly chosen.", partitionedOkay);
if (checkHashJoinStrategies(join, reducer, true)) {
Assert.assertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
} else if (checkHashJoinStrategies(join, reducer, false)) {
Assert.assertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
} else if (checkRepartitionMergeJoin(join, reducer)) {
Assert.assertTrue("Merge join incorrectly chosen", mergeJoinOkay);
} else {
Assert.fail("Plan has no correct hash join or merge join strategies.");
}
} else {
Assert.fail("Plan has neither correct BC join or partitioned join configuration.");
}
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
Example #11
| Source File: TwoInputNode.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public DualInputOperator<?, ?, ?, ?> getOperator() {
return (DualInputOperator<?, ?, ?, ?>) super.getOperator();
}
Example #12
| Source File: TwoInputNode.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExecutionMode) {
// see if there is a hint that dictates which shipping strategy to use for BOTH inputs
final Configuration conf = getOperator().getParameters();
ShipStrategyType preSet1 = null;
ShipStrategyType preSet2 = null;
String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
if (shipStrategy != null) {
if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.FORWARD;
} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.BROADCAST;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.PARTITION_HASH;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.PARTITION_RANGE;
} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
preSet1 = preSet2 = ShipStrategyType.PARTITION_RANDOM;
} else {
throw new CompilerException("Unknown hint for shipping strategy: " + shipStrategy);
}
}
// see if there is a hint that dictates which shipping strategy to use for the FIRST input
shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, null);
if (shipStrategy != null) {
if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
preSet1 = ShipStrategyType.FORWARD;
} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
preSet1 = ShipStrategyType.BROADCAST;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
preSet1 = ShipStrategyType.PARTITION_HASH;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
preSet1 = ShipStrategyType.PARTITION_RANGE;
} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
preSet1 = ShipStrategyType.PARTITION_RANDOM;
} else {
throw new CompilerException("Unknown hint for shipping strategy of input one: " + shipStrategy);
}
}
// see if there is a hint that dictates which shipping strategy to use for the SECOND input
shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, null);
if (shipStrategy != null) {
if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
preSet2 = ShipStrategyType.FORWARD;
} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
preSet2 = ShipStrategyType.BROADCAST;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
preSet2 = ShipStrategyType.PARTITION_HASH;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
preSet2 = ShipStrategyType.PARTITION_RANGE;
} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
preSet2 = ShipStrategyType.PARTITION_RANDOM;
} else {
throw new CompilerException("Unknown hint for shipping strategy of input two: " + shipStrategy);
}
}
// get the predecessors
DualInputOperator<?, ?, ?, ?> contr = getOperator();
Operator<?> leftPred = contr.getFirstInput();
Operator<?> rightPred = contr.getSecondInput();
OptimizerNode pred1;
DagConnection conn1;
if (leftPred == null) {
throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for first input.");
} else {
pred1 = contractToNode.get(leftPred);
conn1 = new DagConnection(pred1, this, defaultExecutionMode);
if (preSet1 != null) {
conn1.setShipStrategy(preSet1);
}
}
// create the connection and add it
this.input1 = conn1;
pred1.addOutgoingConnection(conn1);
OptimizerNode pred2;
DagConnection conn2;
if (rightPred == null) {
throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for second input.");
} else {
pred2 = contractToNode.get(rightPred);
conn2 = new DagConnection(pred2, this, defaultExecutionMode);
if (preSet2 != null) {
conn2.setShipStrategy(preSet2);
}
}
// create the connection and add it
this.input2 = conn2;
pred2.addOutgoingConnection(conn2);
}
Example #13
| Source File: RelationalQueryCompilerTest.java From flink with Apache License 2.0 | 4 votes |
|
private void testQueryGeneric(Plan p, long orderSize, long lineitemSize,
float orderSelectivity, float joinSelectivity,
boolean broadcastOkay, boolean partitionedOkay,
boolean hashJoinFirstOkay, boolean hashJoinSecondOkay, boolean mergeJoinOkay) {
try {
// set statistics
OperatorResolver cr = getContractResolver(p);
GenericDataSourceBase<?, ?> ordersSource = cr.getNode(ORDERS);
GenericDataSourceBase<?, ?> lineItemSource = cr.getNode(LINEITEM);
SingleInputOperator<?, ?, ?> mapper = cr.getNode(MAPPER_NAME);
DualInputOperator<?, ?, ?, ?> joiner = cr.getNode(JOIN_NAME);
setSourceStatistics(ordersSource, orderSize, 100f);
setSourceStatistics(lineItemSource, lineitemSize, 140f);
mapper.getCompilerHints().setAvgOutputRecordSize(16f);
mapper.getCompilerHints().setFilterFactor(orderSelectivity);
joiner.getCompilerHints().setFilterFactor(joinSelectivity);
// compile
final OptimizedPlan plan = compileWithStats(p);
final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);
// get the nodes from the final plan
final SinkPlanNode sink = or.getNode(SINK);
final SingleInputPlanNode reducer = or.getNode(REDUCE_NAME);
final SingleInputPlanNode combiner = reducer.getPredecessor() instanceof SingleInputPlanNode ?
(SingleInputPlanNode) reducer.getPredecessor() : null;
final DualInputPlanNode join = or.getNode(JOIN_NAME);
final SingleInputPlanNode filteringMapper = or.getNode(MAPPER_NAME);
checkStandardStrategies(filteringMapper, join, combiner, reducer, sink);
// check the possible variants and that the variant ia allowed in this specific setting
if (checkBroadcastShipStrategies(join, reducer, combiner)) {
Assert.assertTrue("Broadcast join incorrectly chosen.", broadcastOkay);
if (checkHashJoinStrategies(join, reducer, true)) {
Assert.assertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
} else if (checkHashJoinStrategies(join, reducer, false)) {
Assert.assertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
} else if (checkBroadcastMergeJoin(join, reducer)) {
Assert.assertTrue("Merge join incorrectly chosen", mergeJoinOkay);
} else {
Assert.fail("Plan has no correct hash join or merge join strategies.");
}
}
else if (checkRepartitionShipStrategies(join, reducer, combiner)) {
Assert.assertTrue("Partitioned join incorrectly chosen.", partitionedOkay);
if (checkHashJoinStrategies(join, reducer, true)) {
Assert.assertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
} else if (checkHashJoinStrategies(join, reducer, false)) {
Assert.assertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
} else if (checkRepartitionMergeJoin(join, reducer)) {
Assert.assertTrue("Merge join incorrectly chosen", mergeJoinOkay);
} else {
Assert.fail("Plan has no correct hash join or merge join strategies.");
}
} else {
Assert.fail("Plan has neither correct BC join or partitioned join configuration.");
}
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
Example #14
| Source File: TwoInputNode.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public DualInputOperator<?, ?, ?, ?> getOperator() {
return (DualInputOperator<?, ?, ?, ?>) super.getOperator();
}
Example #15
| Source File: TwoInputNode.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExecutionMode) {
// see if there is a hint that dictates which shipping strategy to use for BOTH inputs
final Configuration conf = getOperator().getParameters();
ShipStrategyType preSet1 = null;
ShipStrategyType preSet2 = null;
String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
if (shipStrategy != null) {
if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.FORWARD;
} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.BROADCAST;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.PARTITION_HASH;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.PARTITION_RANGE;
} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
preSet1 = preSet2 = ShipStrategyType.PARTITION_RANDOM;
} else {
throw new CompilerException("Unknown hint for shipping strategy: " + shipStrategy);
}
}
// see if there is a hint that dictates which shipping strategy to use for the FIRST input
shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, null);
if (shipStrategy != null) {
if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
preSet1 = ShipStrategyType.FORWARD;
} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
preSet1 = ShipStrategyType.BROADCAST;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
preSet1 = ShipStrategyType.PARTITION_HASH;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
preSet1 = ShipStrategyType.PARTITION_RANGE;
} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
preSet1 = ShipStrategyType.PARTITION_RANDOM;
} else {
throw new CompilerException("Unknown hint for shipping strategy of input one: " + shipStrategy);
}
}
// see if there is a hint that dictates which shipping strategy to use for the SECOND input
shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, null);
if (shipStrategy != null) {
if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
preSet2 = ShipStrategyType.FORWARD;
} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
preSet2 = ShipStrategyType.BROADCAST;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
preSet2 = ShipStrategyType.PARTITION_HASH;
} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
preSet2 = ShipStrategyType.PARTITION_RANGE;
} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
preSet2 = ShipStrategyType.PARTITION_RANDOM;
} else {
throw new CompilerException("Unknown hint for shipping strategy of input two: " + shipStrategy);
}
}
// get the predecessors
DualInputOperator<?, ?, ?, ?> contr = getOperator();
Operator<?> leftPred = contr.getFirstInput();
Operator<?> rightPred = contr.getSecondInput();
OptimizerNode pred1;
DagConnection conn1;
if (leftPred == null) {
throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for first input.");
} else {
pred1 = contractToNode.get(leftPred);
conn1 = new DagConnection(pred1, this, defaultExecutionMode);
if (preSet1 != null) {
conn1.setShipStrategy(preSet1);
}
}
// create the connection and add it
this.input1 = conn1;
pred1.addOutgoingConnection(conn1);
OptimizerNode pred2;
DagConnection conn2;
if (rightPred == null) {
throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for second input.");
} else {
pred2 = contractToNode.get(rightPred);
conn2 = new DagConnection(pred2, this, defaultExecutionMode);
if (preSet2 != null) {
conn2.setShipStrategy(preSet2);
}
}
// create the connection and add it
this.input2 = conn2;
pred2.addOutgoingConnection(conn2);
}
