Java Code Examples for org.apache.flink.test.operators.util.CollectionDataSets#getSmall5TupleDataSet()

@Test
public void testCorrectnessOfCrossWithHuge() throws Exception {
	/*
	 * check correctness of crossWithHuge (only correctness of result -> should be the same as with normal cross)
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple2<Integer, String>> crossDs = ds.crossWithHuge(ds2).with(new Tuple5Cross());

	List<Tuple2<Integer, String>> result = crossDs.collect();

	String expected = "0,HalloHallo\n" +
			"1,HalloHallo Welt\n" +
			"2,HalloHallo Welt wie\n" +
			"1,Hallo WeltHallo\n" +
			"2,Hallo WeltHallo Welt\n" +
			"3,Hallo WeltHallo Welt wie\n" +
			"2,Hallo Welt wieHallo\n" +
			"3,Hallo Welt wieHallo Welt\n" +
			"4,Hallo Welt wieHallo Welt wie\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCorrectnessOfDistinctOnTuplesWithKeyFieldSelectorWithNotAllFieldsSelected() throws Exception{
	/*
	 * check correctness of distinct on tuples with key field selector with not all fields selected
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple5<Integer, Long,  Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple1<Integer>> distinctDs = ds.union(ds).distinct(0).project(0);

	List<Tuple1<Integer>> result = distinctDs.collect();

	String expected = "1\n" +
			"2\n";

	compareResultAsTuples(result, expected);
}
 

private void testLeftOuterJoinOnTuplesWithKeyPositions(JoinHint hint) throws Exception {
	/*
	 * UDF Join on tuples with key field positions
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.leftOuterJoin(ds2, hint)
					.where(0)
					.equalTo(0)
					.with(new T3T5FlatJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hallo\n" +
			"Hello,Hallo Welt\n" +
			"Hello,Hallo Welt wie\n" +
			"Hello world,null\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCorretnessOfCrossOnTwoTupleInputs() throws Exception {
	/*
	 * check correctness of cross on two tuple inputs
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple2<Integer, String>> crossDs = ds.cross(ds2).with(new Tuple5Cross());

	List<Tuple2<Integer, String>> result = crossDs.collect();

	String expected = "0,HalloHallo\n" +
			"1,HalloHallo Welt\n" +
			"2,HalloHallo Welt wie\n" +
			"1,Hallo WeltHallo\n" +
			"2,Hallo WeltHallo Welt\n" +
			"3,Hallo WeltHallo Welt wie\n" +
			"2,Hallo Welt wieHallo\n" +
			"3,Hallo Welt wieHallo Welt\n" +
			"4,Hallo Welt wieHallo Welt wie\n";

	compareResultAsTuples(result, expected);
}
 

private void testFullOuterJoinOnTuplesWithKeyPositions(JoinHint hint) throws Exception {
	/*
	 * UDF Join on tuples with key field positions
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.fullOuterJoin(ds2, hint)
					.where(0)
					.equalTo(2)
					.with(new T3T5FlatJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "null,Hallo\n" +
			"Hi,Hallo Welt\n" +
			"Hello,Hallo Welt wie\n" +
			"Hello world,null\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCorrectnessOfCrossATupleInputAndACustomTypeInput() throws Exception {
	/*
	 * check correctness of cross a tuple input and a custom type input
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<CustomType> ds2 = CollectionDataSets.getSmallCustomTypeDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> crossDs = ds.cross(ds2).with(new MixedCross());

	List<Tuple3<Integer, Long, String>> result = crossDs.collect();

	String expected = "2,0,HalloHi\n" +
			"3,0,HalloHello\n" +
			"3,0,HalloHello world\n" +
			"3,0,Hallo WeltHi\n" +
			"4,1,Hallo WeltHello\n" +
			"4,2,Hallo WeltHello world\n" +
			"3,0,Hallo Welt wieHi\n" +
			"4,2,Hallo Welt wieHello\n" +
			"4,4,Hallo Welt wieHello world\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinWithTupleReturningKeySelectors() throws Exception {
	/*
	 * UDF Join on tuples with tuple-returning key selectors
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.fullOuterJoin(ds2)
					.where(new KeySelector3()) //0, 1
					.equalTo(new KeySelector4()) // 0, 4
					.with(new T3T5FlatJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hallo\n" +
			"Hello,Hallo Welt\n" +
			"Hello world,null\n" +
			"null,Hallo Welt wie\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCorrectnessOfDistinctOnTuplesWithKeyFieldSelectorWithNotAllFieldsSelected() throws Exception{
	/*
	 * check correctness of distinct on tuples with key field selector with not all fields selected
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple5<Integer, Long,  Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple1<Integer>> distinctDs = ds.union(ds).distinct(0).project(0);

	List<Tuple1<Integer>> result = distinctDs.collect();

	String expected = "1\n" +
			"2\n";

	compareResultAsTuples(result, expected);
}
 

private void testLeftOuterJoinOnTuplesWithKeyPositions(JoinHint hint) throws Exception {
	/*
	 * UDF Join on tuples with key field positions
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.leftOuterJoin(ds2, hint)
					.where(0)
					.equalTo(0)
					.with(new T3T5FlatJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hallo\n" +
			"Hello,Hallo Welt\n" +
			"Hello,Hallo Welt wie\n" +
			"Hello world,null\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCorrectnessOfCrossATupleInputAndACustomTypeInput() throws Exception {
	/*
	 * check correctness of cross a tuple input and a custom type input
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<CustomType> ds2 = CollectionDataSets.getSmallCustomTypeDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> crossDs = ds.cross(ds2).with(new MixedCross());

	List<Tuple3<Integer, Long, String>> result = crossDs.collect();

	String expected = "2,0,HalloHi\n" +
			"3,0,HalloHello\n" +
			"3,0,HalloHello world\n" +
			"3,0,Hallo WeltHi\n" +
			"4,1,Hallo WeltHello\n" +
			"4,2,Hallo WeltHello world\n" +
			"3,0,Hallo Welt wieHi\n" +
			"4,2,Hallo Welt wieHello\n" +
			"4,4,Hallo Welt wieHello world\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinWithBroadcastSet() throws Exception {
	/*
	 * Join with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple3<String, String, Integer>> joinDs =
			ds1.fullOuterJoin(ds2)
					.where(1)
					.equalTo(4)
					.with(new T3T5BCJoin())
					.withBroadcastSet(intDs, "ints");

	List<Tuple3<String, String, Integer>> result = joinDs.collect();

	String expected = "Hi,Hallo,55\n" +
			"Hi,Hallo Welt wie,55\n" +
			"Hello,Hallo Welt,55\n" +
			"Hello world,Hallo Welt,55\n";

	compareResultAsTuples(result, expected);
}
 

@Test(expected = TableException.class)
public void testRegisterExistingDatasetTable() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	tableEnv.registerDataSet("MyTable", ds);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 =
			CollectionDataSets.getSmall5TupleDataSet(env);
	// Must fail. Name is already used for different table.
	tableEnv.registerDataSet("MyTable", ds2);
}
 

@Test
public void testCorrectnessOfDefaultCross() throws Exception {
	/*
	 * check correctness of default cross
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple2<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>>> crossDs = ds.cross(ds2);

	List<Tuple2<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>>> result = crossDs.collect();

	String expected = "(1,1,Hi),(2,2,1,Hallo Welt,2)\n"
			+
			"(1,1,Hi),(1,1,0,Hallo,1)\n" +
			"(1,1,Hi),(2,3,2,Hallo Welt wie,1)\n" +
			"(2,2,Hello),(2,2,1,Hallo Welt,2)\n" +
			"(2,2,Hello),(1,1,0,Hallo,1)\n" +
			"(2,2,Hello),(2,3,2,Hallo Welt wie,1)\n" +
			"(3,2,Hello world),(2,2,1,Hallo Welt,2)\n" +
			"(3,2,Hello world),(1,1,0,Hallo,1)\n" +
			"(3,2,Hello world),(2,3,2,Hallo Welt wie,1)\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCorrectnessOfDefaultCross() throws Exception {
	/*
	 * check correctness of default cross
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple2<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>>> crossDs = ds.cross(ds2);

	List<Tuple2<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>>> result = crossDs.collect();

	String expected = "(1,1,Hi),(2,2,1,Hallo Welt,2)\n"
			+
			"(1,1,Hi),(1,1,0,Hallo,1)\n" +
			"(1,1,Hi),(2,3,2,Hallo Welt wie,1)\n" +
			"(2,2,Hello),(2,2,1,Hallo Welt,2)\n" +
			"(2,2,Hello),(1,1,0,Hallo,1)\n" +
			"(2,2,Hello),(2,3,2,Hallo Welt wie,1)\n" +
			"(3,2,Hello world),(2,2,1,Hallo Welt,2)\n" +
			"(3,2,Hello world),(1,1,0,Hallo,1)\n" +
			"(3,2,Hello world),(2,3,2,Hallo Welt wie,1)\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testProjectCrossOnATupleInput1() throws Exception{
	/*
	 * project cross on a tuple input 1
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple6<String, Long, String, Integer, Long, Long>> crossDs = ds.cross(ds2)
			.projectFirst(2, 1)
			.projectSecond(3)
			.projectFirst(0)
			.projectSecond(4, 1);

	List<Tuple6<String, Long, String, Integer, Long, Long>> result = crossDs.collect();

	String expected = "Hi,1,Hallo,1,1,1\n" +
			"Hi,1,Hallo Welt,1,2,2\n" +
			"Hi,1,Hallo Welt wie,1,1,3\n" +
			"Hello,2,Hallo,2,1,1\n" +
			"Hello,2,Hallo Welt,2,2,2\n" +
			"Hello,2,Hallo Welt wie,2,1,3\n" +
			"Hello world,2,Hallo,3,1,1\n" +
			"Hello world,2,Hallo Welt,3,2,2\n" +
			"Hello world,2,Hallo Welt wie,3,1,3\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCorrectnessOfCrossWithBroadcastSet() throws Exception {
	/*
	 * check correctness of cross with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple3<Integer, Integer, Integer>> crossDs = ds.cross(ds2).with(new Tuple5CrossBC()).withBroadcastSet(intDs, "ints");

	List<Tuple3<Integer, Integer, Integer>> result = crossDs.collect();

	String expected = "2,0,55\n" +
			"3,0,55\n" +
			"3,0,55\n" +
			"3,0,55\n" +
			"4,1,55\n" +
			"4,2,55\n" +
			"3,0,55\n" +
			"4,2,55\n" +
			"4,4,55\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinWithBroadcastSet() throws Exception {
	/*
	 * Join with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple3<String, String, Integer>> joinDs =
			ds1.fullOuterJoin(ds2)
					.where(1)
					.equalTo(4)
					.with(new T3T5BCJoin())
					.withBroadcastSet(intDs, "ints");

	List<Tuple3<String, String, Integer>> result = joinDs.collect();

	String expected = "Hi,Hallo,55\n" +
			"Hi,Hallo Welt wie,55\n" +
			"Hello,Hallo Welt,55\n" +
			"Hello world,Hallo Welt,55\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinWithBroadcastSet() throws Exception {
	/*
	 * Join with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple3<String, String, Integer>> joinDs =
			ds1.join(ds2)
					.where(1)
					.equalTo(4)
					.with(new T3T5BCJoin())
					.withBroadcastSet(intDs, "ints");

	List<Tuple3<String, String, Integer>> result = joinDs.collect();

	String expected = "Hi,Hallo,55\n" +
			"Hi,Hallo Welt wie,55\n" +
			"Hello,Hallo Welt,55\n" +
			"Hello world,Hallo Welt,55\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinWithBroadcastSet() throws Exception {
	/*
	 * Join with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple3<String, String, Integer>> joinDs =
			ds1.fullOuterJoin(ds2)
					.where(1)
					.equalTo(4)
					.with(new T3T5BCJoin())
					.withBroadcastSet(intDs, "ints");

	List<Tuple3<String, String, Integer>> result = joinDs.collect();

	String expected = "Hi,Hallo,55\n" +
			"Hi,Hallo Welt wie,55\n" +
			"Hello,Hallo Welt,55\n" +
			"Hello world,Hallo Welt,55\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCorrectnessOfDefaultCross() throws Exception {
	/*
	 * check correctness of default cross
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple2<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>>> crossDs = ds.cross(ds2);

	List<Tuple2<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>>> result = crossDs.collect();

	String expected = "(1,1,Hi),(2,2,1,Hallo Welt,2)\n"
			+
			"(1,1,Hi),(1,1,0,Hallo,1)\n" +
			"(1,1,Hi),(2,3,2,Hallo Welt wie,1)\n" +
			"(2,2,Hello),(2,2,1,Hallo Welt,2)\n" +
			"(2,2,Hello),(1,1,0,Hallo,1)\n" +
			"(2,2,Hello),(2,3,2,Hallo Welt wie,1)\n" +
			"(3,2,Hello world),(2,2,1,Hallo Welt,2)\n" +
			"(3,2,Hello world),(1,1,0,Hallo,1)\n" +
			"(3,2,Hello world),(2,3,2,Hallo Welt wie,1)\n";

	compareResultAsTuples(result, expected);
}