org.apache.flink.optimizer.util.OperatorResolver Java Examples

The following examples show how to use org.apache.flink.optimizer.util.OperatorResolver. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KMeansSingleStepTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCompileKMeansSingleStepWithStats() {

	Plan p = getKMeansPlan();
	p.setExecutionConfig(new ExecutionConfig());
	// set the statistics
	OperatorResolver cr = getContractResolver(p);
	GenericDataSourceBase<?, ?> pointsSource = cr.getNode(DATAPOINTS);
	GenericDataSourceBase<?, ?> centersSource = cr.getNode(CENTERS);
	setSourceStatistics(pointsSource, 100L * 1024 * 1024 * 1024, 32f);
	setSourceStatistics(centersSource, 1024 * 1024, 32f);

	OptimizedPlan plan = compileWithStats(p);
	checkPlan(plan);
}
Example #2
Source File: RelationalQueryCompilerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Statistics that push towards a repartition merge join. If the join blows the data volume up significantly,
 * re-exploiting the sorted order is cheaper.
 */
@Test
public void testQueryWithStatsForRepartitionMerge() {
	Plan p = getTPCH3Plan();
	p.setExecutionConfig(defaultExecutionConfig);
	// set compiler hints
	OperatorResolver cr = getContractResolver(p);
	DualInputOperator<?, ?, ?, ?> match = cr.getNode(JOIN_NAME);
	match.getCompilerHints().setFilterFactor(100f);

	testQueryGeneric(100L * 1024 * 1024 * 1024 * 1024, 100L * 1024 * 1024 * 1024 * 1024, 0.01f, 100f, false, true, false, false, true);
}
Example #3
Source File: KMeansSingleStepTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCompileKMeansSingleStepWithStats() {

	Plan p = getKMeansPlan();
	p.setExecutionConfig(new ExecutionConfig());
	// set the statistics
	OperatorResolver cr = getContractResolver(p);
	GenericDataSourceBase<?, ?> pointsSource = cr.getNode(DATAPOINTS);
	GenericDataSourceBase<?, ?> centersSource = cr.getNode(CENTERS);
	setSourceStatistics(pointsSource, 100L * 1024 * 1024 * 1024, 32f);
	setSourceStatistics(centersSource, 1024 * 1024, 32f);

	OptimizedPlan plan = compileWithStats(p);
	checkPlan(plan);
}
Example #4
Source File: RelationalQueryCompilerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Statistics that push towards a repartition merge join. If the join blows the data volume up significantly,
 * re-exploiting the sorted order is cheaper.
 */
@Test
public void testQueryWithStatsForRepartitionMerge() {
	Plan p = getTPCH3Plan();
	p.setExecutionConfig(defaultExecutionConfig);
	// set compiler hints
	OperatorResolver cr = getContractResolver(p);
	DualInputOperator<?, ?, ?, ?> match = cr.getNode(JOIN_NAME);
	match.getCompilerHints().setFilterFactor(100f);

	testQueryGeneric(100L * 1024 * 1024 * 1024 * 1024, 100L * 1024 * 1024 * 1024 * 1024, 0.01f, 100f, false, true, false, false, true);
}
Example #5
Source File: KMeansSingleStepTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCompileKMeansSingleStepWithStats() throws Exception {

	Plan p = getKMeansPlan();
	p.setExecutionConfig(new ExecutionConfig());
	// set the statistics
	OperatorResolver cr = getContractResolver(p);
	GenericDataSourceBase<?, ?> pointsSource = cr.getNode(DATAPOINTS);
	GenericDataSourceBase<?, ?> centersSource = cr.getNode(CENTERS);
	setSourceStatistics(pointsSource, 100L * 1024 * 1024 * 1024, 32f);
	setSourceStatistics(centersSource, 1024 * 1024, 32f);

	OptimizedPlan plan = compileWithStats(p);
	checkPlan(plan);
}
Example #6
Source File: RelationalQueryCompilerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Statistics that push towards a repartition merge join. If the join blows the data volume up significantly,
 * re-exploiting the sorted order is cheaper.
 */
@Test
public void testQueryWithStatsForRepartitionMerge() throws Exception {
	Plan p = getTPCH3Plan();
	p.setExecutionConfig(defaultExecutionConfig);
	// set compiler hints
	OperatorResolver cr = getContractResolver(p);
	DualInputOperator<?, ?, ?, ?> match = cr.getNode(JOIN_NAME);
	match.getCompilerHints().setFilterFactor(100f);

	testQueryGeneric(100L * 1024 * 1024 * 1024 * 1024, 100L * 1024 * 1024 * 1024 * 1024, 0.01f, 100f, false, true, false, false, true);
}
Example #7
Source File: RelationalQueryCompilerTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private void testQueryGeneric(Plan p, long orderSize, long lineitemSize,
		float orderSelectivity, float joinSelectivity,
		boolean broadcastOkay, boolean partitionedOkay,
		boolean hashJoinFirstOkay, boolean hashJoinSecondOkay, boolean mergeJoinOkay) {
	try {
		// set statistics
		OperatorResolver cr = getContractResolver(p);
		GenericDataSourceBase<?, ?> ordersSource = cr.getNode(ORDERS);
		GenericDataSourceBase<?, ?> lineItemSource = cr.getNode(LINEITEM);
		SingleInputOperator<?, ?, ?> mapper = cr.getNode(MAPPER_NAME);
		DualInputOperator<?, ?, ?, ?> joiner = cr.getNode(JOIN_NAME);
		setSourceStatistics(ordersSource, orderSize, 100f);
		setSourceStatistics(lineItemSource, lineitemSize, 140f);
		mapper.getCompilerHints().setAvgOutputRecordSize(16f);
		mapper.getCompilerHints().setFilterFactor(orderSelectivity);
		joiner.getCompilerHints().setFilterFactor(joinSelectivity);

		// compile
		final OptimizedPlan plan = compileWithStats(p);
		final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);

		// get the nodes from the final plan
		final SinkPlanNode sink = or.getNode(SINK);
		final SingleInputPlanNode reducer = or.getNode(REDUCE_NAME);
		final SingleInputPlanNode combiner = reducer.getPredecessor() instanceof SingleInputPlanNode ?
				(SingleInputPlanNode) reducer.getPredecessor() : null;
		final DualInputPlanNode join = or.getNode(JOIN_NAME);
		final SingleInputPlanNode filteringMapper = or.getNode(MAPPER_NAME);

		checkStandardStrategies(filteringMapper, join, combiner, reducer, sink);

		// check the possible variants and that the variant ia allowed in this specific setting
		if (checkBroadcastShipStrategies(join, reducer, combiner)) {
			Assert.assertTrue("Broadcast join incorrectly chosen.", broadcastOkay);

			if (checkHashJoinStrategies(join, reducer, true)) {
				Assert.assertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
			} else if (checkHashJoinStrategies(join, reducer, false)) {
				Assert.assertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
			} else if (checkBroadcastMergeJoin(join, reducer)) {
				Assert.assertTrue("Merge join incorrectly chosen", mergeJoinOkay);
			} else {
				Assert.fail("Plan has no correct hash join or merge join strategies.");
			}
		}
		else if (checkRepartitionShipStrategies(join, reducer, combiner)) {
			Assert.assertTrue("Partitioned join incorrectly chosen.", partitionedOkay);

			if (checkHashJoinStrategies(join, reducer, true)) {
				Assert.assertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
			} else if (checkHashJoinStrategies(join, reducer, false)) {
				Assert.assertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
			} else if (checkRepartitionMergeJoin(join, reducer)) {
				Assert.assertTrue("Merge join incorrectly chosen", mergeJoinOkay);
			} else {
				Assert.fail("Plan has no correct hash join or merge join strategies.");
			}
		} else {
			Assert.fail("Plan has neither correct BC join or partitioned join configuration.");
		}
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
Example #8
Source File: RelationalQueryCompilerTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private void testQueryGeneric(Plan p, long orderSize, long lineitemSize,
		float orderSelectivity, float joinSelectivity,
		boolean broadcastOkay, boolean partitionedOkay,
		boolean hashJoinFirstOkay, boolean hashJoinSecondOkay, boolean mergeJoinOkay) {
	try {
		// set statistics
		OperatorResolver cr = getContractResolver(p);
		GenericDataSourceBase<?, ?> ordersSource = cr.getNode(ORDERS);
		GenericDataSourceBase<?, ?> lineItemSource = cr.getNode(LINEITEM);
		SingleInputOperator<?, ?, ?> mapper = cr.getNode(MAPPER_NAME);
		DualInputOperator<?, ?, ?, ?> joiner = cr.getNode(JOIN_NAME);
		setSourceStatistics(ordersSource, orderSize, 100f);
		setSourceStatistics(lineItemSource, lineitemSize, 140f);
		mapper.getCompilerHints().setAvgOutputRecordSize(16f);
		mapper.getCompilerHints().setFilterFactor(orderSelectivity);
		joiner.getCompilerHints().setFilterFactor(joinSelectivity);

		// compile
		final OptimizedPlan plan = compileWithStats(p);
		final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);

		// get the nodes from the final plan
		final SinkPlanNode sink = or.getNode(SINK);
		final SingleInputPlanNode reducer = or.getNode(REDUCE_NAME);
		final SingleInputPlanNode combiner = reducer.getPredecessor() instanceof SingleInputPlanNode ?
				(SingleInputPlanNode) reducer.getPredecessor() : null;
		final DualInputPlanNode join = or.getNode(JOIN_NAME);
		final SingleInputPlanNode filteringMapper = or.getNode(MAPPER_NAME);

		checkStandardStrategies(filteringMapper, join, combiner, reducer, sink);

		// check the possible variants and that the variant ia allowed in this specific setting
		if (checkBroadcastShipStrategies(join, reducer, combiner)) {
			Assert.assertTrue("Broadcast join incorrectly chosen.", broadcastOkay);

			if (checkHashJoinStrategies(join, reducer, true)) {
				Assert.assertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
			} else if (checkHashJoinStrategies(join, reducer, false)) {
				Assert.assertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
			} else if (checkBroadcastMergeJoin(join, reducer)) {
				Assert.assertTrue("Merge join incorrectly chosen", mergeJoinOkay);
			} else {
				Assert.fail("Plan has no correct hash join or merge join strategies.");
			}
		}
		else if (checkRepartitionShipStrategies(join, reducer, combiner)) {
			Assert.assertTrue("Partitioned join incorrectly chosen.", partitionedOkay);

			if (checkHashJoinStrategies(join, reducer, true)) {
				Assert.assertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
			} else if (checkHashJoinStrategies(join, reducer, false)) {
				Assert.assertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
			} else if (checkRepartitionMergeJoin(join, reducer)) {
				Assert.assertTrue("Merge join incorrectly chosen", mergeJoinOkay);
			} else {
				Assert.fail("Plan has no correct hash join or merge join strategies.");
			}
		} else {
			Assert.fail("Plan has neither correct BC join or partitioned join configuration.");
		}
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
Example #9
Source File: RelationalQueryCompilerTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private void testQueryGeneric(Plan p, long orderSize, long lineitemSize,
		float orderSelectivity, float joinSelectivity,
		boolean broadcastOkay, boolean partitionedOkay,
		boolean hashJoinFirstOkay, boolean hashJoinSecondOkay, boolean mergeJoinOkay) {
	try {
		// set statistics
		OperatorResolver cr = getContractResolver(p);
		GenericDataSourceBase<?, ?> ordersSource = cr.getNode(ORDERS);
		GenericDataSourceBase<?, ?> lineItemSource = cr.getNode(LINEITEM);
		SingleInputOperator<?, ?, ?> mapper = cr.getNode(MAPPER_NAME);
		DualInputOperator<?, ?, ?, ?> joiner = cr.getNode(JOIN_NAME);
		setSourceStatistics(ordersSource, orderSize, 100f);
		setSourceStatistics(lineItemSource, lineitemSize, 140f);
		mapper.getCompilerHints().setAvgOutputRecordSize(16f);
		mapper.getCompilerHints().setFilterFactor(orderSelectivity);
		joiner.getCompilerHints().setFilterFactor(joinSelectivity);

		// compile
		final OptimizedPlan plan = compileWithStats(p);
		final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);

		// get the nodes from the final plan
		final SinkPlanNode sink = or.getNode(SINK);
		final SingleInputPlanNode reducer = or.getNode(REDUCE_NAME);
		final SingleInputPlanNode combiner = reducer.getPredecessor() instanceof SingleInputPlanNode ?
				(SingleInputPlanNode) reducer.getPredecessor() : null;
		final DualInputPlanNode join = or.getNode(JOIN_NAME);
		final SingleInputPlanNode filteringMapper = or.getNode(MAPPER_NAME);

		checkStandardStrategies(filteringMapper, join, combiner, reducer, sink);

		// check the possible variants and that the variant ia allowed in this specific setting
		if (checkBroadcastShipStrategies(join, reducer, combiner)) {
			Assert.assertTrue("Broadcast join incorrectly chosen.", broadcastOkay);

			if (checkHashJoinStrategies(join, reducer, true)) {
				Assert.assertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
			} else if (checkHashJoinStrategies(join, reducer, false)) {
				Assert.assertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
			} else if (checkBroadcastMergeJoin(join, reducer)) {
				Assert.assertTrue("Merge join incorrectly chosen", mergeJoinOkay);
			} else {
				Assert.fail("Plan has no correct hash join or merge join strategies.");
			}
		}
		else if (checkRepartitionShipStrategies(join, reducer, combiner)) {
			Assert.assertTrue("Partitioned join incorrectly chosen.", partitionedOkay);

			if (checkHashJoinStrategies(join, reducer, true)) {
				Assert.assertTrue("Hash join (build orders) incorrectly chosen", hashJoinFirstOkay);
			} else if (checkHashJoinStrategies(join, reducer, false)) {
				Assert.assertTrue("Hash join (build lineitem) incorrectly chosen", hashJoinSecondOkay);
			} else if (checkRepartitionMergeJoin(join, reducer)) {
				Assert.assertTrue("Merge join incorrectly chosen", mergeJoinOkay);
			} else {
				Assert.fail("Plan has no correct hash join or merge join strategies.");
			}
		} else {
			Assert.fail("Plan has neither correct BC join or partitioned join configuration.");
		}
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}