Java Code Examples for org.apache.flink.test.operators.util.CollectionDataSets#getCustomTypeDataSet()

@Test
public void testCorrectnessOfGroupReduceOnCustomTypeWithKeyExtractorAndCombine() throws Exception {
	/*
	 * check correctness of groupReduce on custom type with key extractor and combine
	 */
	org.junit.Assume.assumeTrue(mode != TestExecutionMode.COLLECTION);

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> reduceDs = ds.
			groupBy(new KeySelector3()).reduceGroup(new CustomTypeGroupReduceWithCombine());

	List<CustomType> result = reduceDs.collect();

	String expected = "1,0,test1\n" +
			"2,3,test2\n" +
			"3,12,test3\n" +
			"4,30,test4\n" +
			"5,60,test5\n" +
			"6,105,test6\n";

	compareResultAsText(result, expected);
}
 

@Test
public void testJoinOnATupleInputWithKeyFieldSelectorAndACustomTypeInputWithKeyExtractor()
		throws Exception {
	/*
	 * Join on a tuple input with key field selector and a custom type input with key extractor
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<CustomType> ds2 = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.join(ds2)
					.where(1).equalTo(new KeySelector2())
					.with(new T3CustJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hello\n" +
			"Hello,Hello world\n" +
			"Hello world,Hello world\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCoGroupTwoCustomTypeInputsWithExpressionKeys() throws Exception {
	/*
	 * CoGroup on two custom type inputs using expression keys
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> ds2 = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> coGroupDs = ds.coGroup(ds2).where("myInt").equalTo("myInt").with(new CustomTypeCoGroup());

	List<CustomType> result = coGroupDs.collect();

	String expected = "1,0,test\n" +
			"2,6,test\n" +
			"3,24,test\n" +
			"4,60,test\n" +
			"5,120,test\n" +
			"6,210,test\n";

	compareResultAsText(result, expected);
}
 

@Test
public void testCorrectnessOfDistinctOnCustomTypeWithTypeExtractor() throws Exception {
	/*
	 * check correctness of distinct on custom type with type extractor
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<Tuple1<Integer>> reduceDs = ds
			.distinct(new KeySelector3())
			.map(new Mapper3());

	List<Tuple1<Integer>> result = reduceDs.collect();

	String expected = "1\n" +
			"2\n" +
			"3\n" +
			"4\n" +
			"5\n" +
			"6\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testFilterOnCustomType() throws Exception {
	/*
	 * Test filter on custom type
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> filterDs = ds.
			filter(new Filter6());
	List<CustomType> result = filterDs.collect();

	String expected = "3,3,Hello world, how are you?\n"
			+
			"3,4,I am fine.\n" +
			"3,5,Luke Skywalker\n";

	compareResultAsText(result, expected);
}
 

@Test
public void testCorrectnessOfAllGroupReduceForCustomTypes() throws Exception {
	/*
	 * check correctness of all-groupreduce for custom types
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> reduceDs = ds.reduceGroup(new AllAddingCustomTypeGroupReduce());

	List<CustomType> result = reduceDs.collect();

	String expected = "91,210,Hello!";

	compareResultAsText(result, expected);
}
 

@Test
public void testPojoKeySelectorGroupSort() throws Exception {
	/*
	 * check correctness of sorted groupReduce on custom type with keyselector sorting
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> reduceDs = ds
			.groupBy(new TwoTuplePojoExtractor())
			.sortGroup(new StringPojoExtractor(), Order.DESCENDING)
			.reduceGroup(new CustomTypeSortedGroupReduce());

	List<CustomType> result = reduceDs.collect();

	String expected = "1,0,Hi\n"
			+
			"2,3,Hello world-Hello\n" +
			"3,12,Luke Skywalker-I am fine.-Hello world, how are you?\n" +
			"4,30,Comment#4-Comment#3-Comment#2-Comment#1\n" +
			"5,60,Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" +
			"6,105,Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n";

	compareResultAsText(result, expected);
}
 

@Test
public void testCorrectnessOfGroupReduceOnCustomTypeWithKeyExtractorAndCombine() throws Exception {
	/*
	 * check correctness of groupReduce on custom type with key extractor and combine
	 */
	org.junit.Assume.assumeTrue(mode != TestExecutionMode.COLLECTION);

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> reduceDs = ds.
			groupBy(new KeySelector3()).reduceGroup(new CustomTypeGroupReduceWithCombine());

	List<CustomType> result = reduceDs.collect();

	String expected = "1,0,test1\n" +
			"2,3,test2\n" +
			"3,12,test3\n" +
			"4,30,test4\n" +
			"5,60,test5\n" +
			"6,105,test6\n";

	compareResultAsText(result, expected);
}
 

@Test
public void testCorrectnessOfAllGroupReduceForCustomTypes() throws Exception {
	/*
	 * check correctness of all-groupreduce for custom types
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> reduceDs = ds.reduceGroup(new AllAddingCustomTypeGroupReduce());

	List<CustomType> result = reduceDs.collect();

	String expected = "91,210,Hello!";

	compareResultAsText(result, expected);
}
 

@Test
public void testAllReduceForCustomTypes() throws Exception {
	/*
	 * All-reduce for custom types
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> reduceDs = ds.
			reduce(new AllAddingCustomTypeReduce());

	List<CustomType> result = reduceDs.collect();

	String expected = "91,210,Hello!";

	compareResultAsText(result, expected);
}
 

@Test
public void testPojoKeySelectorGroupSort() throws Exception {
	/*
	 * check correctness of sorted groupReduce on custom type with keyselector sorting
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> reduceDs = ds
			.groupBy(new TwoTuplePojoExtractor())
			.sortGroup(new StringPojoExtractor(), Order.DESCENDING)
			.reduceGroup(new CustomTypeSortedGroupReduce());

	List<CustomType> result = reduceDs.collect();

	String expected = "1,0,Hi\n"
			+
			"2,3,Hello world-Hello\n" +
			"3,12,Luke Skywalker-I am fine.-Hello world, how are you?\n" +
			"4,30,Comment#4-Comment#3-Comment#2-Comment#1\n" +
			"5,60,Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" +
			"6,105,Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n";

	compareResultAsText(result, expected);
}
 

@Test
public void testCorrectnessOfGroupReduceOnCustomTypeWithTypeExtractor() throws Exception {
	/*
	 * check correctness of groupReduce on custom type with type extractor
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> reduceDs = ds.
			groupBy(new KeySelector2()).reduceGroup(new CustomTypeGroupReduce());

	List<CustomType> result = reduceDs.collect();

	String expected = "1,0,Hello!\n" +
			"2,3,Hello!\n" +
			"3,12,Hello!\n" +
			"4,30,Hello!\n" +
			"5,60,Hello!\n" +
			"6,105,Hello!\n";

	compareResultAsText(result, expected);
}
 

@Test
public void testPojoKeySelectorGroupSort() throws Exception {
	/*
	 * check correctness of sorted groupReduce on custom type with keyselector sorting
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> reduceDs = ds
			.groupBy(new TwoTuplePojoExtractor())
			.sortGroup(new StringPojoExtractor(), Order.DESCENDING)
			.reduceGroup(new CustomTypeSortedGroupReduce());

	List<CustomType> result = reduceDs.collect();

	String expected = "1,0,Hi\n"
			+
			"2,3,Hello world-Hello\n" +
			"3,12,Luke Skywalker-I am fine.-Hello world, how are you?\n" +
			"4,30,Comment#4-Comment#3-Comment#2-Comment#1\n" +
			"5,60,Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" +
			"6,105,Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n";

	compareResultAsText(result, expected);
}
 

@Test
public void testCorrectnessOfGroupReduceOnCustomTypeWithKeyExtractorAndCombine() throws Exception {
	/*
	 * check correctness of groupReduce on custom type with key extractor and combine
	 */
	org.junit.Assume.assumeTrue(mode != TestExecutionMode.COLLECTION);

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> reduceDs = ds.
			groupBy(new KeySelector3()).reduceGroup(new CustomTypeGroupReduceWithCombine());

	List<CustomType> result = reduceDs.collect();

	String expected = "1,0,test1\n" +
			"2,3,test2\n" +
			"3,12,test3\n" +
			"4,30,test4\n" +
			"5,60,test5\n" +
			"6,105,test6\n";

	compareResultAsText(result, expected);
}
 

@Test
public void testCoGroupOnATupleInputWithKeyFieldSelectorAndACustomTypeInputWithKeyExtractor()
		throws Exception {
	/*
	 * CoGroup on a tuple input with key field selector and a custom type input with key extractor
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env);
	DataSet<CustomType> ds2 = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> coGroupDs = ds.coGroup(ds2).where(2).equalTo(new
			KeySelector2()).with(new MixedCoGroup());

	List<Tuple3<Integer, Long, String>> result = coGroupDs.collect();

	String expected = "0,1,test\n" +
			"1,2,test\n" +
			"2,5,test\n" +
			"3,15,test\n" +
			"4,33,test\n" +
			"5,63,test\n" +
			"6,109,test\n" +
			"7,4,test\n" +
			"8,4,test\n" +
			"9,4,test\n" +
			"10,5,test\n" +
			"11,5,test\n" +
			"12,5,test\n" +
			"13,5,test\n" +
			"14,5,test\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testTypeConversionFlatMapperCustomToTuple() throws Exception {
	/*
	 * Test type conversion flatmapper (Custom -> Tuple)
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> typeConversionFlatMapDs = ds.
			flatMap(new FlatMapper4());

	List<Tuple3<Integer, Long, String>> result = typeConversionFlatMapDs.collect();

	String expected = "1,0,Hi\n" +
			"2,1,Hello\n" +
			"2,2,Hello world\n" +
			"3,3,Hello world, how are you?\n" +
			"3,4,I am fine.\n" +
			"3,5,Luke Skywalker\n" +
			"4,6,Comment#1\n" +
			"4,7,Comment#2\n" +
			"4,8,Comment#3\n" +
			"4,9,Comment#4\n" +
			"5,10,Comment#5\n" +
			"5,11,Comment#6\n" +
			"5,12,Comment#7\n" +
			"5,13,Comment#8\n" +
			"5,14,Comment#9\n" +
			"6,15,Comment#10\n" +
			"6,16,Comment#11\n" +
			"6,17,Comment#12\n" +
			"6,18,Comment#13\n" +
			"6,19,Comment#14\n" +
			"6,20,Comment#15\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testMapperOnCustomLowercaseString() throws Exception {
	/*
	 * Test mapper on Custom - lowercase myString
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> customMapDs = ds.
			map(new Mapper6());

	List<CustomType> result = customMapDs.collect();

	String expected = "1,0,hi\n" +
			"2,1,hello\n" +
			"2,2,hello world\n" +
			"3,3,hello world, how are you?\n" +
			"3,4,i am fine.\n" +
			"3,5,luke skywalker\n" +
			"4,6,comment#1\n" +
			"4,7,comment#2\n" +
			"4,8,comment#3\n" +
			"4,9,comment#4\n" +
			"5,10,comment#5\n" +
			"5,11,comment#6\n" +
			"5,12,comment#7\n" +
			"5,13,comment#8\n" +
			"5,14,comment#9\n" +
			"6,15,comment#10\n" +
			"6,16,comment#11\n" +
			"6,17,comment#12\n" +
			"6,18,comment#13\n" +
			"6,19,comment#14\n" +
			"6,20,comment#15\n";

	compareResultAsText(result, expected);
}
 

@Test
public void testTypeConversionMapperCustomToTuple() throws Exception {
	/*
	 * Test type conversion mapper (Custom -> Tuple)
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> typeConversionMapDs = ds.
			map(new Mapper3());

	List<Tuple3<Integer, Long, String>> result = typeConversionMapDs.collect();

	String expected = "1,0,Hi\n" +
			"2,1,Hello\n" +
			"2,2,Hello world\n" +
			"3,3,Hello world, how are you?\n" +
			"3,4,I am fine.\n" +
			"3,5,Luke Skywalker\n" +
			"4,6,Comment#1\n" +
			"4,7,Comment#2\n" +
			"4,8,Comment#3\n" +
			"4,9,Comment#4\n" +
			"5,10,Comment#5\n" +
			"5,11,Comment#6\n" +
			"5,12,Comment#7\n" +
			"5,13,Comment#8\n" +
			"5,14,Comment#9\n" +
			"6,15,Comment#10\n" +
			"6,16,Comment#11\n" +
			"6,17,Comment#12\n" +
			"6,18,Comment#13\n" +
			"6,19,Comment#14\n" +
			"6,20,Comment#15\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCoGroupOnACustomTypeWithKeyExtractorAndATupleInputWithKeyFieldSelector()
		throws Exception {
	/*
	 * CoGroup on a tuple input with key field selector and a custom type input with key extractor
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env);
	DataSet<CustomType> ds2 = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<CustomType> coGroupDs = ds2.coGroup(ds).where(new KeySelector3()).equalTo(2).with
			(new MixedCoGroup2());

	List<CustomType> result = coGroupDs.collect();

	String expected = "0,1,test\n" +
			"1,2,test\n" +
			"2,5,test\n" +
			"3,15,test\n" +
			"4,33,test\n" +
			"5,63,test\n" +
			"6,109,test\n" +
			"7,4,test\n" +
			"8,4,test\n" +
			"9,4,test\n" +
			"10,5,test\n" +
			"11,5,test\n" +
			"12,5,test\n" +
			"13,5,test\n" +
			"14,5,test\n";

	compareResultAsText(result, expected);
}
 

@Test
public void testTypeConversionFlatMapperCustomToTuple() throws Exception {
	/*
	 * Test type conversion flatmapper (Custom -> Tuple)
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> typeConversionFlatMapDs = ds.
			flatMap(new FlatMapper4());

	List<Tuple3<Integer, Long, String>> result = typeConversionFlatMapDs.collect();

	String expected = "1,0,Hi\n" +
			"2,1,Hello\n" +
			"2,2,Hello world\n" +
			"3,3,Hello world, how are you?\n" +
			"3,4,I am fine.\n" +
			"3,5,Luke Skywalker\n" +
			"4,6,Comment#1\n" +
			"4,7,Comment#2\n" +
			"4,8,Comment#3\n" +
			"4,9,Comment#4\n" +
			"5,10,Comment#5\n" +
			"5,11,Comment#6\n" +
			"5,12,Comment#7\n" +
			"5,13,Comment#8\n" +
			"5,14,Comment#9\n" +
			"6,15,Comment#10\n" +
			"6,16,Comment#11\n" +
			"6,17,Comment#12\n" +
			"6,18,Comment#13\n" +
			"6,19,Comment#14\n" +
			"6,20,Comment#15\n";

	compareResultAsTuples(result, expected);
}