Java Code Examples for org.apache.flink.test.operators.util.CollectionDataSets#getIntegerDataSet()

@Test
public void testRichFilterOnStringTupleField() throws Exception {
	/*
	 * Test filter on String tuple field.
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> ints = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
			filter(new RichFilter1()).withBroadcastSet(ints, "ints");
	List<Tuple3<Integer, Long, String>> result = filterDs.collect();

	String expected = "1,1,Hi\n" +
			"2,2,Hello\n" +
			"3,2,Hello world\n" +
			"4,3,Hello world, how are you?\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCorrectnessOfDistinctOnAtomic() throws Exception {
	/*
	 * check correctness of distinct on Integers
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Integer> ds = CollectionDataSets.getIntegerDataSet(env);
	DataSet<Integer> reduceDs = ds.distinct();

	List<Integer> result = reduceDs.collect();

	String expected = "1\n2\n3\n4\n5";

	compareResultAsText(result, expected);
}
 

@Test
public void testReduceWithBroadcastSet() throws Exception {
	/*
	 * Reduce with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.
			groupBy(1).reduce(new BCTuple3Reduce()).withBroadcastSet(intDs, "ints");

	List<Tuple3<Integer, Long, String>> result = reduceDs.collect();

	String expected = "1,1,Hi\n" +
			"5,2,55\n" +
			"15,3,55\n" +
			"34,4,55\n" +
			"65,5,55\n" +
			"111,6,55\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCorrectnessOfDistinctOnAtomic() throws Exception {
	/*
	 * check correctness of distinct on Integers
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Integer> ds = CollectionDataSets.getIntegerDataSet(env);
	DataSet<Integer> reduceDs = ds.distinct();

	List<Integer> result = reduceDs.collect();

	String expected = "1\n2\n3\n4\n5";

	compareResultAsText(result, expected);
}
 

@Test
public void testCoGroupWithBroadcastSet() throws Exception {
	/*
	 * Reduce with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
	DataSet<Tuple3<Integer, Integer, Integer>> coGroupDs = ds.coGroup(ds2).where(0).equalTo(0).with(new Tuple5CoGroupBC()).withBroadcastSet(intDs, "ints");

	List<Tuple3<Integer, Integer, Integer>> result = coGroupDs.collect();

	String expected = "1,0,55\n" +
			"2,6,55\n" +
			"3,24,55\n" +
			"4,60,55\n" +
			"5,120,55\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testRichFilterOnStringTupleField() throws Exception {
	/*
	 * Test filter on String tuple field.
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> ints = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
			filter(new RichFilter1()).withBroadcastSet(ints, "ints");
	List<Tuple3<Integer, Long, String>> result = filterDs.collect();

	String expected = "1,1,Hi\n" +
			"2,2,Hello\n" +
			"3,2,Hello world\n" +
			"4,3,Hello world, how are you?\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCorrectnessOfGroupReduceWithBroadcastSet() throws Exception {
	/*
	 * check correctness of groupReduce with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.
			groupBy(1).reduceGroup(new BCTuple3GroupReduce()).withBroadcastSet(intDs, "ints");

	List<Tuple3<Integer, Long, String>> result = reduceDs.collect();

	String expected = "1,1,55\n" +
			"5,2,55\n" +
			"15,3,55\n" +
			"34,4,55\n" +
			"65,5,55\n" +
			"111,6,55\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCorrectnessOfGroupReduceWithBroadcastSet() throws Exception {
	/*
	 * check correctness of groupReduce with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.
			groupBy(1).reduceGroup(new BCTuple3GroupReduce()).withBroadcastSet(intDs, "ints");

	List<Tuple3<Integer, Long, String>> result = reduceDs.collect();

	String expected = "1,1,55\n" +
			"5,2,55\n" +
			"15,3,55\n" +
			"34,4,55\n" +
			"65,5,55\n" +
			"111,6,55\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinWithBroadcastSet() throws Exception {
	/*
	 * Join with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple3<String, String, Integer>> joinDs =
			ds1.join(ds2)
					.where(1)
					.equalTo(4)
					.with(new T3T5BCJoin())
					.withBroadcastSet(intDs, "ints");

	List<Tuple3<String, String, Integer>> result = joinDs.collect();

	String expected = "Hi,Hallo,55\n" +
			"Hi,Hallo Welt wie,55\n" +
			"Hello,Hallo Welt,55\n" +
			"Hello world,Hallo Welt,55\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testAggregatorWithoutParameterForIterate() throws Exception {
	/*
	 * Test aggregator without parameter for iterate
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(parallelism);

	DataSet<Integer> initialSolutionSet = CollectionDataSets.getIntegerDataSet(env);
	IterativeDataSet<Integer> iteration = initialSolutionSet.iterate(MAX_ITERATIONS);

	// register aggregator
	LongSumAggregator aggr = new LongSumAggregator();
	iteration.registerAggregator(NEGATIVE_ELEMENTS_AGGR, aggr);

	// register convergence criterion
	iteration.registerAggregationConvergenceCriterion(NEGATIVE_ELEMENTS_AGGR, aggr,
			new NegativeElementsConvergenceCriterion());

	DataSet<Integer> updatedDs = iteration.map(new SubtractOneMap());
	List<Integer> result = iteration.closeWith(updatedDs).collect();
	Collections.sort(result);

	List<Integer> expected = Arrays.asList(-3, -2, -2, -1, -1, -1, 0, 0, 0, 0, 1, 1, 1, 1, 1);

	assertEquals(expected, result);
}
 

@Test
public void testConvergenceCriterionWithParameterForIterate() throws Exception {
	/*
	 * Test convergence criterion with parameter for iterate
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(parallelism);

	DataSet<Integer> initialSolutionSet = CollectionDataSets.getIntegerDataSet(env);
	IterativeDataSet<Integer> iteration = initialSolutionSet.iterate(MAX_ITERATIONS);

	// register aggregator
	LongSumAggregator aggr = new LongSumAggregator();
	iteration.registerAggregator(NEGATIVE_ELEMENTS_AGGR, aggr);

	// register convergence criterion
	iteration.registerAggregationConvergenceCriterion(NEGATIVE_ELEMENTS_AGGR, aggr,
			new NegativeElementsConvergenceCriterionWithParam(3));

	DataSet<Integer> updatedDs = iteration.map(new SubtractOneMap());
	List<Integer> result = iteration.closeWith(updatedDs).collect();
	Collections.sort(result);

	List<Integer> expected = Arrays.asList(-3, -2, -2, -1, -1, -1, 0, 0, 0, 0, 1, 1, 1, 1, 1);

	assertEquals(expected, result);
}
 

@Test
public void testMapWithBroadcastSet() throws Exception {
	/*
	 * Test map with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> ints = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> bcMapDs = ds.
			map(new RichMapper1()).withBroadcastSet(ints, "ints");
	List<Tuple3<Integer, Long, String>> result = bcMapDs.collect();

	String expected = "55,1,Hi\n" +
			"55,2,Hello\n" +
			"55,2,Hello world\n" +
			"55,3,Hello world, how are you?\n" +
			"55,3,I am fine.\n" +
			"55,3,Luke Skywalker\n" +
			"55,4,Comment#1\n" +
			"55,4,Comment#2\n" +
			"55,4,Comment#3\n" +
			"55,4,Comment#4\n" +
			"55,5,Comment#5\n" +
			"55,5,Comment#6\n" +
			"55,5,Comment#7\n" +
			"55,5,Comment#8\n" +
			"55,5,Comment#9\n" +
			"55,6,Comment#10\n" +
			"55,6,Comment#11\n" +
			"55,6,Comment#12\n" +
			"55,6,Comment#13\n" +
			"55,6,Comment#14\n" +
			"55,6,Comment#15\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testAggregatorWithoutParameterForIterate() throws Exception {
	/*
	 * Test aggregator without parameter for iterate
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(parallelism);

	DataSet<Integer> initialSolutionSet = CollectionDataSets.getIntegerDataSet(env);
	IterativeDataSet<Integer> iteration = initialSolutionSet.iterate(MAX_ITERATIONS);

	// register aggregator
	LongSumAggregator aggr = new LongSumAggregator();
	iteration.registerAggregator(NEGATIVE_ELEMENTS_AGGR, aggr);

	// register convergence criterion
	iteration.registerAggregationConvergenceCriterion(NEGATIVE_ELEMENTS_AGGR, aggr,
			new NegativeElementsConvergenceCriterion());

	DataSet<Integer> updatedDs = iteration.map(new SubtractOneMap());
	List<Integer> result = iteration.closeWith(updatedDs).collect();
	Collections.sort(result);

	List<Integer> expected = Arrays.asList(-3, -2, -2, -1, -1, -1, 0, 0, 0, 0, 1, 1, 1, 1, 1);

	assertEquals(expected, result);
}
 

@Test
public void testFlatMapWithBroadcastSet() throws Exception {
	/*
	 * Test flatmap with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> ints = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> bcFlatMapDs = ds.
			flatMap(new RichFlatMapper1()).withBroadcastSet(ints, "ints");
	List<Tuple3<Integer, Long, String>> result = bcFlatMapDs.collect();

	String expected = "55,1,Hi\n" +
			"55,2,Hello\n" +
			"55,2,Hello world\n" +
			"55,3,Hello world, how are you?\n" +
			"55,3,I am fine.\n" +
			"55,3,Luke Skywalker\n" +
			"55,4,Comment#1\n" +
			"55,4,Comment#2\n" +
			"55,4,Comment#3\n" +
			"55,4,Comment#4\n" +
			"55,5,Comment#5\n" +
			"55,5,Comment#6\n" +
			"55,5,Comment#7\n" +
			"55,5,Comment#8\n" +
			"55,5,Comment#9\n" +
			"55,6,Comment#10\n" +
			"55,6,Comment#11\n" +
			"55,6,Comment#12\n" +
			"55,6,Comment#13\n" +
			"55,6,Comment#14\n" +
			"55,6,Comment#15\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testJoinWithBroadcastSet() throws Exception {
	/*
	 * Join with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple3<String, String, Integer>> joinDs =
			ds1.fullOuterJoin(ds2)
					.where(1)
					.equalTo(4)
					.with(new T3T5BCJoin())
					.withBroadcastSet(intDs, "ints");

	List<Tuple3<String, String, Integer>> result = joinDs.collect();

	String expected = "Hi,Hallo,55\n" +
			"Hi,Hallo Welt wie,55\n" +
			"Hello,Hallo Welt,55\n" +
			"Hello world,Hallo Welt,55\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testCorrectnessOfCrossWithBroadcastSet() throws Exception {
	/*
	 * check correctness of cross with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple3<Integer, Integer, Integer>> crossDs = ds.cross(ds2).with(new Tuple5CrossBC()).withBroadcastSet(intDs, "ints");

	List<Tuple3<Integer, Integer, Integer>> result = crossDs.collect();

	String expected = "2,0,55\n" +
			"3,0,55\n" +
			"3,0,55\n" +
			"3,0,55\n" +
			"4,1,55\n" +
			"4,2,55\n" +
			"3,0,55\n" +
			"4,2,55\n" +
			"4,4,55\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testAggregatorWithParameterForIterate() throws Exception {
	/*
	 * Test aggregator with parameter for iterate
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(parallelism);

	DataSet<Integer> initialSolutionSet = CollectionDataSets.getIntegerDataSet(env);
	IterativeDataSet<Integer> iteration = initialSolutionSet.iterate(MAX_ITERATIONS);

	// register aggregator
	LongSumAggregatorWithParameter aggr = new LongSumAggregatorWithParameter(0);
	iteration.registerAggregator(NEGATIVE_ELEMENTS_AGGR, aggr);

	// register convergence criterion
	iteration.registerAggregationConvergenceCriterion(NEGATIVE_ELEMENTS_AGGR, aggr,
			new NegativeElementsConvergenceCriterion());

	DataSet<Integer> updatedDs = iteration.map(new SubtractOneMapWithParam());
	List<Integer> result = iteration.closeWith(updatedDs).collect();
	Collections.sort(result);

	List<Integer> expected = Arrays.asList(-3, -2, -2, -1, -1, -1, 0, 0, 0, 0, 1, 1, 1, 1, 1);

	assertEquals(expected, result);
}
 

@Test
public void testJoinWithBroadcastSet() throws Exception {
	/*
	 * Join with broadcast set
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.getSmall5TupleDataSet(env);
	DataSet<Tuple3<String, String, Integer>> joinDs =
			ds1.join(ds2)
					.where(1)
					.equalTo(4)
					.with(new T3T5BCJoin())
					.withBroadcastSet(intDs, "ints");

	List<Tuple3<String, String, Integer>> result = joinDs.collect();

	String expected = "Hi,Hallo,55\n" +
			"Hi,Hallo Welt wie,55\n" +
			"Hello,Hallo Welt,55\n" +
			"Hello world,Hallo Welt,55\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testIntSortingParallelism1() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> ds = CollectionDataSets.getIntegerDataSet(env);
	ds.writeAsText(resultPath).sortLocalOutput("*", Order.DESCENDING).setParallelism(1);

	env.execute();

	String expected = "5\n5\n5\n5\n5\n4\n4\n4\n4\n3\n3\n3\n2\n2\n1\n";
	compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath);

}
 

@Test
public void testIntSortingParallelism1() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Integer> ds = CollectionDataSets.getIntegerDataSet(env);
	ds.writeAsText(resultPath).sortLocalOutput("*", Order.DESCENDING).setParallelism(1);

	env.execute();

	String expected = "5\n5\n5\n5\n5\n4\n4\n4\n4\n3\n3\n3\n2\n2\n1\n";
	compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath);

}