Java Code Examples for org.apache.flink.api.java.DataSet#groupBy()

@Test
@SuppressWarnings("serial")
public void testGroupByKeySelector3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	this.customTypeData.add(new CustomType());

	try {
		DataSet<CustomType> customDs = env.fromCollection(customTypeData);
		// should not work
		customDs.groupBy(
				new KeySelector<GroupingTest.CustomType, CustomType>() {
					@Override
					public CustomType getKey(CustomType value) {
						return value;
					}
				});
	} catch (Exception e) {
		Assert.fail();
	}
}
 

@Test
public void testForkingReduceOnKeyedDataset() throws Exception {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);

	// creates the input data and distributes them evenly among the available downstream tasks
	DataSet<Tuple3<String, Integer, Boolean>> input = createKeyedInput(env);

	UnsortedGrouping<Tuple3<String, Integer, Boolean>> counts = input.groupBy(0);

	DataSet<Tuple3<String, Integer, Boolean>> r1 = counts.reduceGroup(new KeyedCombReducer());
	DataSet<Tuple3<String, Integer, Boolean>> r2 = counts.reduceGroup(new KeyedGroupCombReducer());

	List<Tuple3<String, Integer, Boolean>> actual = r1.union(r2).collect();
	String expected = "k1,6,true\n" +
		"k2,4,true\n" +
		"k1,6,true\n" +
		"k2,4,true\n";
	compareResultAsTuples(actual, expected);
}
 

@Test(expected = InvalidProgramException.class)
@SuppressWarnings("serial")
public void testGroupByKeySelector5() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	this.customTypeData.add(new CustomType());

	DataSet<CustomType> customDs = env.fromCollection(customTypeData);
	// should not work
	customDs.groupBy(
			new KeySelector<GroupingTest.CustomType, CustomType2>() {
				@Override
				public CustomType2 getKey(CustomType value) {
					return new CustomType2();
				}
			});
}
 

@Test(expected = InvalidProgramException.class)
@SuppressWarnings("serial")
public void testGroupByKeySelector5() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	this.customTypeData.add(new CustomType());

	DataSet<CustomType> customDs = env.fromCollection(customTypeData);
	// should not work
	customDs.groupBy(
			new KeySelector<GroupingTest.CustomType, CustomType2>() {
				@Override
				public CustomType2 getKey(CustomType value) {
					return new CustomType2();
				}
			});
}
 

@Test
public void testForkingReduceOnKeyedDatasetWithSelection() throws Exception {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);

	// creates the input data and distributes them evenly among the available downstream tasks
	DataSet<Tuple3<String, Integer, Boolean>> input = createKeyedInput(env);

	UnsortedGrouping<Tuple3<String, Integer, Boolean>> counts = input.groupBy(new KeySelectorX());

	DataSet<Tuple3<String, Integer, Boolean>> r1 = counts.reduceGroup(new KeyedCombReducer());
	DataSet<Tuple3<String, Integer, Boolean>> r2 = counts.reduceGroup(new KeyedGroupCombReducer());

	List<Tuple3<String, Integer, Boolean>> actual = r1.union(r2).collect();
	String expected = "k1,6,true\n" +
		"k2,4,true\n" +
		"k1,6,true\n" +
		"k2,4,true\n";

	compareResultAsTuples(actual, expected);
}
 

@Test
@SuppressWarnings("serial")
public void testGroupByKeySelector2() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	this.customTypeData.add(new CustomType());

	try {
		DataSet<CustomType> customDs = env.fromCollection(customTypeData);
		// should work
		customDs.groupBy(
				new KeySelector<GroupingTest.CustomType, Tuple2<Integer, Long>>() {
					@Override
					public Tuple2<Integer, Long> getKey(CustomType value) {
						return new Tuple2<Integer, Long>(value.myInt, value.myLong);
				}
		});
	} catch (Exception e) {
		Assert.fail();
	}
}
 

@Test
@SuppressWarnings("serial")
public void testGroupByKeySelector1() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	this.customTypeData.add(new CustomType());

	try {
		DataSet<CustomType> customDs = env.fromCollection(customTypeData);
		// should work
		customDs.groupBy(
				new KeySelector<GroupingTest.CustomType, Long>() {

					@Override
					public Long getKey(CustomType value) {
						return value.myLong;
				}
		});
	} catch (Exception e) {
		Assert.fail();
	}
}
 

@Test
@SuppressWarnings("serial")
public void testGroupByKeySelector1() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	this.customTypeData.add(new CustomType());

	try {
		DataSet<CustomType> customDs = env.fromCollection(customTypeData);
		// should work
		customDs.groupBy(
				new KeySelector<GroupingTest.CustomType, Long>() {

					@Override
					public Long getKey(CustomType value) {
						return value.myLong;
				}
		});
	} catch (Exception e) {
		Assert.fail();
	}
}
 

@Test
public void testForkingReduceOnKeyedDataset() throws Exception {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);

	// creates the input data and distributes them evenly among the available downstream tasks
	DataSet<Tuple3<String, Integer, Boolean>> input = createKeyedInput(env);

	UnsortedGrouping<Tuple3<String, Integer, Boolean>> counts = input.groupBy(0);

	DataSet<Tuple3<String, Integer, Boolean>> r1 = counts.reduceGroup(new KeyedCombReducer());
	DataSet<Tuple3<String, Integer, Boolean>> r2 = counts.reduceGroup(new KeyedGroupCombReducer());

	List<Tuple3<String, Integer, Boolean>> actual = r1.union(r2).collect();
	String expected = "k1,6,true\n" +
		"k2,4,true\n" +
		"k1,6,true\n" +
		"k2,4,true\n";
	compareResultAsTuples(actual, expected);
}
 

@Test(expected = InvalidProgramException.class)
public void testGroupAtomicTypeWithInvalid1() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Integer> dataSet = env.fromElements(0, 1, 2, 3);

	dataSet.groupBy("*", "invalidField");
}
 

@Test(expected = InvalidProgramException.class)
public void testGroupAtomicTypeWithInvalid2() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Integer> dataSet = env.fromElements(0, 1, 2, 3);

	dataSet.groupBy("invalidField");
}
 

@Test(expected = IllegalArgumentException.class)
public void testGroupByKeyExpressions4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<CustomType> ds = env.fromCollection(customTypeData);

	// should not work, key out of tuple bounds
	ds.groupBy("myNonExistent");
}
 

@Test(expected = InvalidProgramException.class)
public void testGroupByKeyExpressions2() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Long> longDs = env.fromCollection(emptyLongData, BasicTypeInfo.LONG_TYPE_INFO);
	// should not work: groups on basic type
	longDs.groupBy("myInt");
}
 

@Test(expected = InvalidProgramException.class)
public void testGroupAtomicTypeWithInvalid1() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Integer> dataSet = env.fromElements(0, 1, 2, 3);

	dataSet.groupBy("*", "invalidField");
}
 

@Test(expected = IndexOutOfBoundsException.class)
public void testGroupByKeyFields5() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);

	// should not work, negative field position
	tupleDs.groupBy(-1);
}
 

@Test(expected = IndexOutOfBoundsException.class)
public void testGroupByKeyFields4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);

	// should not work, key out of tuple bounds
	tupleDs.groupBy(5);
}
 

@Test(expected = InvalidProgramException.class)
public void testGroupAtomicTypeWithInvalid2() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Integer> dataSet = env.fromElements(0, 1, 2, 3);

	dataSet.groupBy("invalidField");
}
 

@Test(expected = InvalidProgramException.class)
public void testGroupAtomicTypeWithInvalid3() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<ArrayList<Integer>> dataSet = env.fromElements(new ArrayList<Integer>());

	dataSet.groupBy("*");
}
 

private DataSet<Tuple> translateGroupBy(DataSet<Tuple> input, FlowNode node, int dop) {

		GroupBy groupBy = (GroupBy) node.getSourceElements().iterator().next();

		Scope outScope = getOutScope(node);
		List<Scope> inScopes = getInputScopes(node, groupBy);

		Fields outFields;
		if(outScope.isEvery()) {
			outFields = outScope.getOutGroupingFields();
		}
		else {
			outFields = outScope.getOutValuesFields();
		}
		registerKryoTypes(outFields);

		// get input scope
		Scope inScope = inScopes.get(0);

		// get grouping keys
		Fields groupKeyFields = groupBy.getKeySelectors().get(inScope.getName());
		// get group sorting keys
		Fields sortKeyFields = groupBy.getSortingSelectors().get(inScope.getName());

		String[] groupKeys = registerKeyFields(input, groupKeyFields);
		String[] sortKeys = null;
		if (sortKeyFields != null) {
			sortKeys = registerKeyFields(input, sortKeyFields);
		}
		Order sortOrder = groupBy.isSortReversed() ? Order.DESCENDING : Order.ASCENDING;

		if(sortOrder == Order.DESCENDING) {
			// translate groupBy with inverse sort order
			return translateInverseSortedGroupBy(input, node, dop, groupKeys, sortKeys, outFields);
		}
		else if(groupKeys == null || groupKeys.length == 0) {
			// translate key-less (global) groupBy
			return translateGlobalGroupBy(input, node, dop, sortKeys, sortOrder, outFields);
		}
		else {

			UnsortedGrouping<Tuple> grouping = input
					.groupBy(groupKeys);

			if(sortKeys != null && sortKeys.length > 0) {
				// translate groupBy with group sorting

				SortedGrouping<Tuple> sortedGrouping = grouping
						.sortGroup(sortKeys[0], Order.ASCENDING);
				for(int i=1; i<sortKeys.length; i++) {
					sortedGrouping = sortedGrouping
							.sortGroup(sortKeys[i], Order.DESCENDING);
				}

				return sortedGrouping
						.reduceGroup(new GroupByReducer(node))
						.returns(new TupleTypeInfo(outFields))
						.withParameters(this.getFlinkNodeConfig(node))
						.setParallelism(dop)
						.name("reduce-" + node.getID());
			}
			else {
				// translate groupBy without group sorting

				return grouping
						.reduceGroup(new GroupByReducer(node))
						.returns(new TupleTypeInfo(outFields))
						.withParameters(this.getFlinkNodeConfig(node))
						.setParallelism(dop)
						.name("reduce-" + node.getID());
			}
		}

	}
 

@Test(expected = InvalidProgramException.class)
public void testGroupByKeyExpressions3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	this.customTypeData.add(new CustomType());

	DataSet<CustomType> customDs = env.fromCollection(customTypeData);
	// should not work: tuple selector on custom type
	customDs.groupBy(0);

}