Java Code Examples for org.apache.flink.test.operators.util.CollectionDataSets#getSmall3TupleDataSet()

@Test
public void testFlatJoinWithTypeInformationTypeHint() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Integer> resultDs = ds1
		.join(ds2)
		.where(0)
		.equalTo(0)
		.with(new FlatJoiner<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>, Integer>())
		.returns(BasicTypeInfo.INT_TYPE_INFO);
	List<Integer> result = resultDs.collect();

	String expectedResult = "2\n" +
		"3\n" +
		"1\n";

	compareResultAsText(result, expectedResult);
}
 

@Test
public void testJoinWithHuge() throws Exception {
	/*
	 * Join with Huge
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
	DataSet<Tuple2<String, String>> joinDs = ds1.joinWithHuge(ds2)
			.where(1)
			.equalTo(1)
			.with(new T3T5FlatJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hallo\n" +
			"Hello,Hallo Welt\n" +
			"Hello world,Hallo Welt\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinOnATupleInputWithKeyFieldSelectorAndACustomTypeInputWithKeyExtractor()
		throws Exception {
	/*
	 * Join on a tuple input with key field selector and a custom type input with key extractor
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<CustomType> ds2 = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.join(ds2)
					.where(1).equalTo(new KeySelector2())
					.with(new T3CustJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hello\n" +
			"Hello,Hello world\n" +
			"Hello world,Hello world\n";

	compareResultAsTuples(result, expected);
}
 

private void testRightOuterJoinOnTuplesWithKeyPositions(JoinHint hint) throws Exception {
	/*
	 * UDF Join on tuples with key field positions
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.rightOuterJoin(ds2, hint)
					.where(1)
					.equalTo(1)
					.with(new T3T5FlatJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hallo\n" +
			"Hello,Hallo Welt\n" +
			"null,Hallo Welt wie\n" +
			"Hello world,Hallo Welt\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinOnTuplesWithCompositeKeyPositions() throws Exception {
	/*
	 * UDF Join on tuples with multiple key field positions
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.fullOuterJoin(ds2)
					.where(0, 1)
					.equalTo(0, 4)
					.with(new T3T5FlatJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hallo\n" +
			"Hello,Hallo Welt\n" +
			"Hello world,null\n" +
			"null,Hallo Welt wie\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinWithTypeInformationTypeHint() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().disableSysoutLogging();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Integer> resultDs = ds1
		.join(ds2)
		.where(0)
		.equalTo(0)
		.with(new Joiner<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>, Integer>())
		.returns(BasicTypeInfo.INT_TYPE_INFO);
	List<Integer> result = resultDs.collect();

	String expectedResult = "2\n" +
		"3\n" +
		"1\n";

	compareResultAsText(result, expectedResult);
}
 

@Test
public void testUnsortedGroupReduceWithTypeInformationTypeHint() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().disableSysoutLogging();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Integer> resultDs = ds
		.groupBy(0)
		.reduceGroup(new GroupReducer<Tuple3<Integer, Long, String>, Integer>())
		.returns(BasicTypeInfo.INT_TYPE_INFO);
	List<Integer> result = resultDs.collect();

	String expectedResult = "2\n" +
		"3\n" +
		"1\n";

	compareResultAsText(result, expectedResult);
}
 

@Test
public void testJoinOnATupleInputWithKeyFieldSelectorAndACustomTypeInputWithKeyExtractor()
		throws Exception {
	/*
	 * Join on a tuple input with key field selector and a custom type input with key extractor
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<CustomType> ds2 = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.join(ds2)
					.where(1).equalTo(new KeySelector2())
					.with(new T3CustJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hello\n" +
			"Hello,Hello world\n" +
			"Hello world,Hello world\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinWithTiny() throws Exception {
	/*
	 * Join with Tiny
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.joinWithTiny(ds2)
					.where(1)
					.equalTo(1)
					.with(new T3T5FlatJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hallo\n" +
			"Hello,Hallo Welt\n" +
			"Hello world,Hallo Welt\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinThatReturnsTheRightInputObject() throws Exception {
	/*
	 * Join that returns the right input object
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> joinDs =
			ds1.join(ds2)
					.where(1)
					.equalTo(1)
					.with(new RightReturningJoin());

	List<Tuple5<Integer, Long, Integer, String, Long>> result = joinDs.collect();

	String expected = "1,1,0,Hallo,1\n" +
			"2,2,1,Hallo Welt,2\n" +
			"2,2,1,Hallo Welt,2\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinWithTiny() throws Exception {
	/*
	 * Join with Tiny
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.joinWithTiny(ds2)
					.where(1)
					.equalTo(1)
					.with(new T3T5FlatJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hallo\n" +
			"Hello,Hallo Welt\n" +
			"Hello world,Hallo Welt\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinWithMixedKeyTypes1() throws Exception {
	/*
	 * Join on a tuple input with key field selector and a custom type input with key extractor
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds1 = CollectionDataSets.getSmallCustomTypeDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.fullOuterJoin(ds2)
					.where(new KeySelector1())
					.equalTo(0)
					.with(new CustT3Join());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hi\n" +
			"Hello,Hello\n" +
			"Hello world,Hello\n" +
			"null,Hello world\n";

	compareResultAsTuples(result, expected);

}
 

@Test
public void testJoinWithHuge() throws Exception {
	/*
	 * Join with Huge
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
	DataSet<Tuple2<String, String>> joinDs = ds1.joinWithHuge(ds2)
			.where(1)
			.equalTo(1)
			.with(new T3T5FlatJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hallo\n" +
			"Hello,Hallo Welt\n" +
			"Hello world,Hallo Welt\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testDefaultJoinOnTuples() throws Exception {
	/*
	 * Default Join on tuples
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
	DataSet<Tuple2<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>>> joinDs =
			ds1.join(ds2)
					.where(0)
					.equalTo(2);

	List<Tuple2<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>>> result = joinDs.collect();

	String expected = "(1,1,Hi),(2,2,1,Hallo Welt,2)\n" +
			"(2,2,Hello),(2,3,2,Hallo Welt wie,1)\n" +
			"(3,2,Hello world),(3,4,3,Hallo Welt wie gehts?,2)\n";

	compareResultAsTuples(result, expected);

}
 

@Test
public void testJoin() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);

	tableEnv.registerDataSet("t1", ds1, "a, b, c");
	tableEnv.registerDataSet("t2", ds2, "d, e, f, g, h");

	String sqlQuery = "SELECT c, g FROM t1, t2 WHERE b = e";
	Table result = tableEnv.sqlQuery(sqlQuery);

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();
	String expected = "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n";
	compareResultAsText(results, expected);
}
 

@Test
public void testProjectJoinOnATuple2Input() throws Exception {
	/*
	 * Project join on a tuple input 2
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
	DataSet<Tuple6<String, String, Long, Long, Long, Integer>> joinDs =
			ds1.join(ds2)
					.where(1)
					.equalTo(1)
					.projectSecond(3)
					.projectFirst(2, 1)
					.projectSecond(4, 1)
					.projectFirst(0);

	List<Tuple6<String, String, Long, Long, Long, Integer>> result = joinDs.collect();

	String expected = "Hallo,Hi,1,1,1,1\n" +
			"Hallo Welt,Hello,2,2,2,2\n" +
			"Hallo Welt,Hello world,2,2,2,3\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCoGroupWithAtomicType2() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Integer> ds1 = env.fromElements(0, 1, 2);
	DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> coGroupDs = ds1.coGroup(ds2).where("*").equalTo(0).with(new CoGroupAtomic2());

	List<Tuple3<Integer, Long, String>> result = coGroupDs.collect();

	String expected = "(1,1,Hi)\n" +
		"(2,2,Hello)";

	compareResultAsText(result, expected);
}
 

@Test
public void testProjectJoinOnATuple2Input() throws Exception {
	/*
	 * Project join on a tuple input 2
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
	DataSet<Tuple6<String, String, Long, Long, Long, Integer>> joinDs =
			ds1.join(ds2)
					.where(1)
					.equalTo(1)
					.projectSecond(3)
					.projectFirst(2, 1)
					.projectSecond(4, 1)
					.projectFirst(0);

	List<Tuple6<String, String, Long, Long, Long, Integer>> result = joinDs.collect();

	String expected = "Hallo,Hi,1,1,1,1\n" +
			"Hallo Welt,Hello,2,2,2,2\n" +
			"Hallo Welt,Hello world,2,2,2,3\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinWithAtomicType2() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> ds1 = env.fromElements(1, 2);
	DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env);

	DataSet<Tuple2<Integer, Tuple3<Integer, Long, String>>> joinDs = ds1.join(ds2).where("*").equalTo(0);

	List<Tuple2<Integer, Tuple3<Integer, Long, String>>> result = joinDs.collect();

	String expected = "1,(1,1,Hi)\n" +
			"2,(2,2,Hello)";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinWithAtomicType1() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Integer> ds2 = env.fromElements(1, 2);

	DataSet<Tuple2<Tuple3<Integer, Long, String>, Integer>> joinDs = ds1.join(ds2).where(0).equalTo("*");

	List<Tuple2<Tuple3<Integer, Long, String>, Integer>> result = joinDs.collect();

	String expected = "(1,1,Hi),1\n" +
			"(2,2,Hello),2";

	compareResultAsTuples(result, expected);
}