org.apache.flink.optimizer.plan.DualInputPlanNode Java Examples

/**
 * This test makes sure that only a HYBRIDHASH on the static path is transformed to the cached variant
 */
@Test
public void testLeftSideCountercheck() {
	try {
		
		Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
		
		OptimizedPlan oPlan = compileNoStats(plan);

		OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
		DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
		
		// verify correct join strategy
		assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, innerJoin.getDriverStrategy());
		assertEquals(TempMode.CACHED, innerJoin.getInput1().getTempMode());
		assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
	
		new JobGraphGenerator().compileJobGraph(oPlan);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Test errored: " + e.getMessage());
	}
}
 

private boolean checkRepartitionMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) {
	if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) {
		// driver keys
		Assert.assertEquals(set0, join.getKeysForInput1());
		Assert.assertEquals(set0, join.getKeysForInput2());

		// local strategies
		Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy());
		Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy());
		Assert.assertEquals(LocalStrategy.NONE, reducer.getInput().getLocalStrategy());

		// local strategy keys
		Assert.assertEquals(set01, join.getInput1().getLocalStrategyKeys());
		Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys());
		Assert.assertTrue(join.getInput1().getLocalStrategySortOrder()[0] == join.getInput2().getLocalStrategySortOrder()[0]);
		Assert.assertEquals(set01, reducer.getKeys(0));
		Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), reducer.getSortOrders(0)));
		return true;
	} else {
		return false;
	}
}
 

@Override
public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
	DriverStrategy strategy;
	
	if(!in1.isOnDynamicPath() && in2.isOnDynamicPath()) {
		// sanity check that the first input is cached and remove that cache
		if (!in1.getTempMode().isCached()) {
			throw new CompilerException("No cache at point where static and dynamic parts meet.");
		}
		in1.setTempMode(in1.getTempMode().makeNonCached());
		strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED;
	}
	else {
		strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST;
	}
	return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
}
 

private boolean checkBroadcastMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) {
	if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) {
		// driver keys
		Assert.assertEquals(set0, join.getKeysForInput1());
		Assert.assertEquals(set0, join.getKeysForInput2());

		// local strategies
		Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy());
		Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy());
		Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy());

		// local strategy keys
		Assert.assertEquals(set0, join.getInput1().getLocalStrategyKeys());
		Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys());
		Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), join.getInput2().getLocalStrategySortOrder()));
		Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys());
		Assert.assertEquals(set01, reducer.getKeys(0));
		Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0)));
		return true;
	} else {
		return false;
	}
}
 

/**
 * This test makes sure that only a HYBRIDHASH on the static path is transformed to the cached variant
 */
@Test
public void testRightSideCountercheck() {
	try {
		
		Plan plan = getTestPlanRightStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
		
		OptimizedPlan oPlan = compileNoStats(plan);

		OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
		DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
		
		// verify correct join strategy
		assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, innerJoin.getDriverStrategy()); 
		assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
		assertEquals(TempMode.CACHED, innerJoin.getInput2().getTempMode());
	
		new JobGraphGenerator().compileJobGraph(oPlan);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Test errored: " + e.getMessage());
	}
}
 

/**
 * This tests whether a HYBRIDHASH_BUILD_FIRST is correctly transformed to a HYBRIDHASH_BUILD_FIRST_CACHED
 * when inside of an iteration an on the static path
 */
@Test
public void testLeftSide() {
	try {
		
		Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
		
		OptimizedPlan oPlan = compileNoStats(plan);

		OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
		DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
		
		// verify correct join strategy
		assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED, innerJoin.getDriverStrategy());
		assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
		assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
	
		new JobGraphGenerator().compileJobGraph(oPlan);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Test errored: " + e.getMessage());
	}
}
 

@Override
public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
	DriverStrategy strategy;
	
	if (!in2.isOnDynamicPath() && in1.isOnDynamicPath()) {
		// sanity check that the first input is cached and remove that cache
		if (!in2.getTempMode().isCached()) {
			throw new CompilerException("No cache at point where static and dynamic parts meet.");
		}
		
		in2.setTempMode(in2.getTempMode().makeNonCached());
		strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED;
	}
	else {
		strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND;
	}
	return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
}
 

@Override
public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
	DriverStrategy strategy;
	
	if(!in1.isOnDynamicPath() && in2.isOnDynamicPath()) {
		// sanity check that the first input is cached and remove that cache
		if (!in1.getTempMode().isCached()) {
			throw new CompilerException("No cache at point where static and dynamic parts meet.");
		}
		in1.setTempMode(in1.getTempMode().makeNonCached());
		strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED;
	}
	else {
		strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST;
	}
	return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
}
 

/**
 * This test makes sure that only a HYBRIDHASH on the static path is transformed to the cached variant
 */
@Test
public void testRightSideCountercheck() {
	try {
		
		Plan plan = getTestPlanRightStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
		
		OptimizedPlan oPlan = compileNoStats(plan);

		OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
		DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
		
		// verify correct join strategy
		assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, innerJoin.getDriverStrategy()); 
		assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
		assertEquals(TempMode.CACHED, innerJoin.getInput2().getTempMode());
	
		new JobGraphGenerator().compileJobGraph(oPlan);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Test errored: " + e.getMessage());
	}
}
 

private boolean checkRepartitionMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) {
	if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) {
		// driver keys
		Assert.assertEquals(set0, join.getKeysForInput1());
		Assert.assertEquals(set0, join.getKeysForInput2());

		// local strategies
		Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy());
		Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy());
		Assert.assertEquals(LocalStrategy.NONE, reducer.getInput().getLocalStrategy());

		// local strategy keys
		Assert.assertEquals(set01, join.getInput1().getLocalStrategyKeys());
		Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys());
		Assert.assertTrue(join.getInput1().getLocalStrategySortOrder()[0] == join.getInput2().getLocalStrategySortOrder()[0]);
		Assert.assertEquals(set01, reducer.getKeys(0));
		Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), reducer.getSortOrders(0)));
		return true;
	} else {
		return false;
	}
}
 

private boolean checkBroadcastMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) {
	if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) {
		// driver keys
		Assert.assertEquals(set0, join.getKeysForInput1());
		Assert.assertEquals(set0, join.getKeysForInput2());

		// local strategies
		Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy());
		Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy());
		Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy());

		// local strategy keys
		Assert.assertEquals(set0, join.getInput1().getLocalStrategyKeys());
		Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys());
		Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), join.getInput2().getLocalStrategySortOrder()));
		Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys());
		Assert.assertEquals(set01, reducer.getKeys(0));
		Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0)));
		return true;
	} else {
		return false;
	}
}
 

@Test
public void testCrossWithSmall() {
	// construct the plan
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);
	DataSet<Long> set1 = env.generateSequence(0,1);
	DataSet<Long> set2 = env.generateSequence(0,1);

	set1.crossWithTiny(set2).name("Cross")
			.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());

	try {
		Plan plan = env.createProgramPlan();
		OptimizedPlan oPlan = compileWithStats(plan);
		OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
		
		DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
		Channel in1 = crossPlanNode.getInput1();
		Channel in2 = crossPlanNode.getInput2();
		
		assertEquals(ShipStrategyType.FORWARD, in1.getShipStrategy());
		assertEquals(ShipStrategyType.BROADCAST, in2.getShipStrategy());
	} catch(CompilerException ce) {
		ce.printStackTrace();
		fail("The Flink optimizer is unable to compile this plan correctly.");
	}
}
 

@Test
public void testCrossWithLarge() {
	// construct the plan
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);
	DataSet<Long> set1 = env.generateSequence(0,1);
	DataSet<Long> set2 = env.generateSequence(0,1);

	set1.crossWithHuge(set2).name("Cross")
			.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());

	try {
		Plan plan = env.createProgramPlan();
		OptimizedPlan oPlan = compileNoStats(plan);
		OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
		
		DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
		Channel in1 = crossPlanNode.getInput1();
		Channel in2 = crossPlanNode.getInput2();
		
		assertEquals(ShipStrategyType.BROADCAST, in1.getShipStrategy());
		assertEquals(ShipStrategyType.FORWARD, in2.getShipStrategy());
	} catch(CompilerException ce) {
		ce.printStackTrace();
		fail("The pact compiler is unable to compile this plan correctly.");
	}
}
 

private boolean checkBroadcastShipStrategies(DualInputPlanNode join, SingleInputPlanNode reducer,
		SingleInputPlanNode combiner) {
	if (ShipStrategyType.BROADCAST == join.getInput1().getShipStrategy() &&
		ShipStrategyType.FORWARD == join.getInput2().getShipStrategy() &&
		ShipStrategyType.PARTITION_HASH == reducer.getInput().getShipStrategy()) {

		// check combiner
		Assert.assertNotNull("Plan should have a combiner", combiner);
		Assert.assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
		return true;
	} else {
		return false;
	}
}
 

/**
 * Verifies that a robust repartitioning plan with a hash join is created in the absence of statistics.
 */
@Test
public void testQueryNoStatistics() {
	try {
		Plan p = getTPCH3Plan();
		p.setExecutionConfig(defaultExecutionConfig);
		// compile
		final OptimizedPlan plan = compileNoStats(p);

		final OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(plan);

		// get the nodes from the final plan
		final SinkPlanNode sink = or.getNode(SINK);
		final SingleInputPlanNode reducer = or.getNode(REDUCE_NAME);
		final SingleInputPlanNode combiner = reducer.getPredecessor() instanceof SingleInputPlanNode ?
				(SingleInputPlanNode) reducer.getPredecessor() : null;
		final DualInputPlanNode join = or.getNode(JOIN_NAME);
		final SingleInputPlanNode filteringMapper = or.getNode(MAPPER_NAME);

		// verify the optimizer choices
		checkStandardStrategies(filteringMapper, join, combiner, reducer, sink);
		Assert.assertTrue(checkRepartitionShipStrategies(join, reducer, combiner));
		Assert.assertTrue(checkHashJoinStrategies(join, reducer, true) || checkHashJoinStrategies(join, reducer, false));
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

private void checkStandardStrategies(SingleInputPlanNode map, DualInputPlanNode join, SingleInputPlanNode combiner,
		SingleInputPlanNode reducer, SinkPlanNode sink) {
	// check ship strategies that are always fix
	Assert.assertEquals(ShipStrategyType.FORWARD, map.getInput().getShipStrategy());
	Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());

	// check the driver strategies that are always fix
	Assert.assertEquals(DriverStrategy.FLAT_MAP, map.getDriverStrategy());
	Assert.assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reducer.getDriverStrategy());
	Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy());
	if (combiner != null) {
		Assert.assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy());
		Assert.assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy());
	}
}
 

@Test
public void testOptimizerChoosesTest() {
	try {
		DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.OPTIMIZER_CHOOSES);
		assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
		assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
		assertTrue(DriverStrategy.HYBRIDHASH_BUILD_FIRST == node.getDriverStrategy() ||
				DriverStrategy.HYBRIDHASH_BUILD_SECOND == node.getDriverStrategy());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getClass().getSimpleName() + ": " + e.getMessage());
	}
}
 

/**
 * Tests join program with replicated data source behind flatMap.
 */
@Test
public void checkJoinWithReplicatedSourceInputBehindFlatMap() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	TupleTypeInfo<Tuple1<String>> typeInfo = TupleTypeInfo.getBasicTupleTypeInfo(String.class);
	ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
			new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new TupleCsvInputFormat<Tuple1<String>>(new Path("/some/path"), typeInfo));

	DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
	DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);

	DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
			.flatMap(new IdFlatMap())
			.join(source2).where("*").equalTo("*")
			.writeAsText("/some/newpath");

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	// when join should have forward strategy on both sides
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();

	ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
	ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();

	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
}
 

@Test
public void testCoGroupWithKeySelectors() {
	try {
		final Partitioner<Integer> partitioner = new TestPartitionerInt();
		
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		
		DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
		DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
		
		input1
			.coGroup(input2)
			.where(new Pojo2KeySelector()).equalTo(new Pojo3KeySelector())
			.withPartitioner(partitioner)
			.with(new DummyCoGroupFunction<Pojo2, Pojo3>())
			.output(new DiscardingOutputFormat<Tuple2<Pojo2, Pojo3>>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
		
		assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
		assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
		assertEquals(partitioner, join.getInput1().getPartitioner());
		assertEquals(partitioner, join.getInput2().getPartitioner());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

/**
 * Tests cross program with replicated data source behind map and filter.
 */
@Test
public void checkCrossWithReplicatedSourceInputBehindMap() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	TupleTypeInfo<Tuple1<String>> typeInfo = TupleTypeInfo.getBasicTupleTypeInfo(String.class);
	ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
			new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new TupleCsvInputFormat<Tuple1<String>>(new Path("/some/path"), typeInfo));

	DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
	DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);

	DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
			.map(new IdMap())
			.filter(new NoFilter())
			.cross(source2)
			.writeAsText("/some/newpath");

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	// when cross should have forward strategy on both sides
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	DualInputPlanNode crossNode = (DualInputPlanNode) sinkNode.getPredecessor();

	ShipStrategyType crossIn1 = crossNode.getInput1().getShipStrategy();
	ShipStrategyType crossIn2 = crossNode.getInput2().getShipStrategy();

	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn1);
	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn2);
}
 

/**
 * Tests cross program with replicated data source.
 */
@Test
public void checkCrossWithReplicatedSourceInput() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	TupleTypeInfo<Tuple1<String>> typeInfo = TupleTypeInfo.getBasicTupleTypeInfo(String.class);
	ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
			new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new TupleCsvInputFormat<Tuple1<String>>(new Path("/some/path"), typeInfo));

	DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
	DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);

	DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
			.cross(source2)
			.writeAsText("/some/newpath");

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	// when cross should have forward strategy on both sides
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	DualInputPlanNode crossNode = (DualInputPlanNode) sinkNode.getPredecessor();

	ShipStrategyType crossIn1 = crossNode.getInput1().getShipStrategy();
	ShipStrategyType crossIn2 = crossNode.getInput2().getShipStrategy();

	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn1);
	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn2);
}
 

/**
 * Tests join program with replicated data source behind multiple map ops.
 */
@Test
public void checkJoinWithReplicatedSourceInputBehindMultiMaps() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	TupleTypeInfo<Tuple1<String>> typeInfo = TupleTypeInfo.getBasicTupleTypeInfo(String.class);
	ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
			new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new TupleCsvInputFormat<Tuple1<String>>(new Path("/some/path"), typeInfo));

	DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
	DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);

	DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
			.filter(new NoFilter())
			.mapPartition(new IdPMap())
			.flatMap(new IdFlatMap())
			.map(new IdMap())
			.join(source2).where("*").equalTo("*")
			.writeAsText("/some/newpath");

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	// when join should have forward strategy on both sides
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();

	ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
	ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();

	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
}
 

/**
 * Tests join program with replicated data source behind map partition.
 */
@Test
public void checkJoinWithReplicatedSourceInputBehindMapPartition() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	TupleTypeInfo<Tuple1<String>> typeInfo = TupleTypeInfo.getBasicTupleTypeInfo(String.class);
	ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
			new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new TupleCsvInputFormat<Tuple1<String>>(new Path("/some/path"), typeInfo));

	DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
	DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);

	DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
			.mapPartition(new IdPMap())
			.join(source2).where("*").equalTo("*")
			.writeAsText("/some/newpath");

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	// when join should have forward strategy on both sides
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();

	ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
	ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();

	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
}
 

/**
 * Tests join program with replicated data source behind filter.
 */
@Test
public void checkJoinWithReplicatedSourceInputBehindFilter() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	TupleTypeInfo<Tuple1<String>> typeInfo = TupleTypeInfo.getBasicTupleTypeInfo(String.class);
	ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
			new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new TupleCsvInputFormat<Tuple1<String>>(new Path("/some/path"), typeInfo));

	DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
	DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);

	DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
			.filter(new NoFilter())
			.join(source2).where("*").equalTo("*")
			.writeAsText("/some/newpath");

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	// when join should have forward strategy on both sides
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();

	ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
	ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();

	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
}
 

/**
 * Tests join program with replicated data source behind map.
 */
@Test
public void checkJoinWithReplicatedSourceInputBehindMap() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	TupleTypeInfo<Tuple1<String>> typeInfo = TupleTypeInfo.getBasicTupleTypeInfo(String.class);
	ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
			new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new TupleCsvInputFormat<Tuple1<String>>(new Path("/some/path"), typeInfo));

	DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
	DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);

	DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
			.map(new IdMap())
			.join(source2).where("*").equalTo("*")
			.writeAsText("/some/newpath");

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	// when join should have forward strategy on both sides
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();

	ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
	ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();

	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
}
 

/**
 * Tests join program with replicated data source.
 */
@Test
public void checkJoinWithReplicatedSourceInput() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	TupleTypeInfo<Tuple1<String>> typeInfo = TupleTypeInfo.getBasicTupleTypeInfo(String.class);
	ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
			new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new TupleCsvInputFormat<Tuple1<String>>(new Path("/some/path"), typeInfo));

	DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
	DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);

	DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
			.join(source2).where("*").equalTo("*")
			.writeAsText("/some/newpath");

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	// when join should have forward strategy on both sides
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();

	ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
	ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();

	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
	Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
}
 

@Test
public void testOptimizerChoosesTest() {
	try {
		DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.OPTIMIZER_CHOOSES);
		assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
		assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
		assertTrue(DriverStrategy.HYBRIDHASH_BUILD_FIRST == node.getDriverStrategy() ||
				DriverStrategy.HYBRIDHASH_BUILD_SECOND == node.getDriverStrategy());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getClass().getSimpleName() + ": " + e.getMessage());
	}
}
 

@Test
public void testPartitionSortMergeTest() {
	try {
		DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.REPARTITION_SORT_MERGE);
		assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
		assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
		assertEquals(DriverStrategy.INNER_MERGE, node.getDriverStrategy());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getClass().getSimpleName() + ": " + e.getMessage());
	}
}
 

@Test
public void testNoPartialSolutionFoundTwoInputOperator() {
	try {
		SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");

		SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1");
		SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
		
		Channel toMap1 = new Channel(source1);
		toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap1.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
		
		Channel toMap2 = new Channel(source2);
		toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap2.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
		
		Channel toJoin1 = new Channel(map1);
		Channel toJoin2 = new Channel(map2);
		
		toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toJoin1.setLocalStrategy(LocalStrategy.NONE);
		toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toJoin2.setLocalStrategy(LocalStrategy.NONE);
		
		DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
		
		FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
		assertEquals(NO_PARTIAL_SOLUTION, report);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

private boolean checkRepartitionShipStrategies(DualInputPlanNode join, SingleInputPlanNode reducer,
		SingleInputPlanNode combiner) {
	if (ShipStrategyType.PARTITION_HASH == join.getInput1().getShipStrategy() &&
		ShipStrategyType.PARTITION_HASH == join.getInput2().getShipStrategy() &&
		ShipStrategyType.FORWARD == reducer.getInput().getShipStrategy()) {

		// check combiner
		Assert.assertNull("Plan should not have a combiner", combiner);
		return true;
	} else {
		return false;
	}
}