org.apache.flink.optimizer.testfunctions.Top1GroupReducer Java Examples

The following examples show how to use org.apache.flink.optimizer.testfunctions.Top1GroupReducer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: BranchingPlansCompilerTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testMultipleIterations() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(100);
	
	DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
	
	DataSet<String> reduced = input
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>());
		
	IterativeDataSet<String> iteration1 = input.iterate(100);
	IterativeDataSet<String> iteration2 = input.iterate(20);
	IterativeDataSet<String> iteration3 = input.iterate(17);
	
	iteration1.closeWith(iteration1.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc1"))
			.output(new DiscardingOutputFormat<String>());
	iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(reduced, "bc2"))
			.output(new DiscardingOutputFormat<String>());
	iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()).withBroadcastSet(reduced, "bc3"))
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #2

Source File: BranchingPlansCompilerTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testMultipleIterationsWithClosueBCVars() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(100);

	DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
		
	IterativeDataSet<String> iteration1 = input.iterate(100);
	IterativeDataSet<String> iteration2 = input.iterate(20);
	IterativeDataSet<String> iteration3 = input.iterate(17);
	
	
	iteration1.closeWith(iteration1.map(new IdentityMapper<String>()))
			.output(new DiscardingOutputFormat<String>());
	iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()))
			.output(new DiscardingOutputFormat<String>());
	iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()))
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #3

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

5 votes

@Test
public void testMultipleIterations() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(100);
	
	DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
	
	DataSet<String> reduced = input
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>());
		
	IterativeDataSet<String> iteration1 = input.iterate(100);
	IterativeDataSet<String> iteration2 = input.iterate(20);
	IterativeDataSet<String> iteration3 = input.iterate(17);
	
	iteration1.closeWith(iteration1.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc1"))
			.output(new DiscardingOutputFormat<String>());
	iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(reduced, "bc2"))
			.output(new DiscardingOutputFormat<String>());
	iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()).withBroadcastSet(reduced, "bc3"))
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #4

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

5 votes

@Test
public void testMultipleIterationsWithClosueBCVars() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(100);

	DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
		
	IterativeDataSet<String> iteration1 = input.iterate(100);
	IterativeDataSet<String> iteration2 = input.iterate(20);
	IterativeDataSet<String> iteration3 = input.iterate(17);
	
	
	iteration1.closeWith(iteration1.map(new IdentityMapper<String>()))
			.output(new DiscardingOutputFormat<String>());
	iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()))
			.output(new DiscardingOutputFormat<String>());
	iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()))
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #5

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

5 votes

@Test
public void testMultipleIterations() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(100);
	
	DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
	
	DataSet<String> reduced = input
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>());
		
	IterativeDataSet<String> iteration1 = input.iterate(100);
	IterativeDataSet<String> iteration2 = input.iterate(20);
	IterativeDataSet<String> iteration3 = input.iterate(17);
	
	iteration1.closeWith(iteration1.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc1"))
			.output(new DiscardingOutputFormat<String>());
	iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(reduced, "bc2"))
			.output(new DiscardingOutputFormat<String>());
	iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()).withBroadcastSet(reduced, "bc3"))
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #6

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

5 votes

@Test
public void testMultipleIterationsWithClosueBCVars() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(100);

	DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
		
	IterativeDataSet<String> iteration1 = input.iterate(100);
	IterativeDataSet<String> iteration2 = input.iterate(20);
	IterativeDataSet<String> iteration3 = input.iterate(17);
	
	
	iteration1.closeWith(iteration1.map(new IdentityMapper<String>()))
			.output(new DiscardingOutputFormat<String>());
	iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()))
			.output(new DiscardingOutputFormat<String>());
	iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()))
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #7

Source File: BranchingPlansCompilerTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * 
 * <pre>

 *              (SINK A)
 *                  |    (SINK B)    (SINK C)
 *                CROSS    /          /
 *               /     \   |  +------+
 *              /       \  | /
 *          REDUCE      MATCH2
 *             |    +---/    \
 *              \  /          |
 *               MAP          |
 *                |           |
 *             COGROUP      MATCH1
 *             /     \     /     \
 *        (SRC A)    (SRC B)    (SRC C)
 * </pre>
 */
@Test
public void testBranchingWithMultipleDataSinks() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(DEFAULT_PARALLELISM);

		DataSet<Tuple2<Long, Long>> sourceA = env.generateSequence(1, 10000000)
				.map(new Duplicator<Long>());

		DataSet<Tuple2<Long, Long>> sourceB = env.generateSequence(1, 10000000)
				.map(new Duplicator<Long>());

		DataSet<Tuple2<Long, Long>> sourceC = env.generateSequence(1, 10000000)
				.map(new Duplicator<Long>());

		DataSet<Tuple2<Long, Long>> mapped = sourceA.coGroup(sourceB)
				.where(0).equalTo(1)
				.with(new CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>>() {
						@Override
						public void coGroup(Iterable<Tuple2<Long, Long>> first,
												Iterable<Tuple2<Long, Long>> second,
												Collector<Tuple2<Long, Long>> out) {
						  }
				})
				.map(new IdentityMapper<Tuple2<Long, Long>>());

		DataSet<Tuple2<Long, Long>> joined = sourceB.join(sourceC)
				.where(0).equalTo(1)
				.with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());

		DataSet<Tuple2<Long, Long>> joined2 = mapped.join(joined)
				.where(1).equalTo(1)
				.with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());

		DataSet<Tuple2<Long, Long>> reduced = mapped
				.groupBy(1)
				.reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>());

		reduced.cross(joined2)
				.output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());

		joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
		joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());

		Plan plan = env.createProgramPlan();
		OptimizedPlan oPlan = compileNoStats(plan);
		new JobGraphGenerator().compileJobGraph(oPlan);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #8

Source File: BranchingPlansCompilerTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testBranchingBroadcastVariable() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(100);

	DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1");
	DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2");
	DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3");
	
	DataSet<String> result1 = input1
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>())
				.withBroadcastSet(input3, "bc");
	
	DataSet<String> result2 = input2
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>())
				.withBroadcastSet(input3, "bc");
	
	result1.join(result2)
			.where(new IdentityKeyExtractor<String>())
			.equalTo(new IdentityKeyExtractor<String>())
			.with(new RichJoinFunction<String, String, String>() {
				@Override
				public String join(String first, String second) {
					return null;
				}
			})
			.withBroadcastSet(input3, "bc1")
			.withBroadcastSet(input1, "bc2")
			.withBroadcastSet(result1, "bc3")
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #9

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

4 votes

/**
 * 
 * <pre>

 *              (SINK A)
 *                  |    (SINK B)    (SINK C)
 *                CROSS    /          /
 *               /     \   |  +------+
 *              /       \  | /
 *          REDUCE      MATCH2
 *             |    +---/    \
 *              \  /          |
 *               MAP          |
 *                |           |
 *             COGROUP      MATCH1
 *             /     \     /     \
 *        (SRC A)    (SRC B)    (SRC C)
 * </pre>
 */
@Test
public void testBranchingWithMultipleDataSinks() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(DEFAULT_PARALLELISM);

		DataSet<Tuple2<Long, Long>> sourceA = env.generateSequence(1, 10000000)
				.map(new Duplicator<Long>());

		DataSet<Tuple2<Long, Long>> sourceB = env.generateSequence(1, 10000000)
				.map(new Duplicator<Long>());

		DataSet<Tuple2<Long, Long>> sourceC = env.generateSequence(1, 10000000)
				.map(new Duplicator<Long>());

		DataSet<Tuple2<Long, Long>> mapped = sourceA.coGroup(sourceB)
				.where(0).equalTo(1)
				.with(new CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>>() {
						@Override
						public void coGroup(Iterable<Tuple2<Long, Long>> first,
												Iterable<Tuple2<Long, Long>> second,
												Collector<Tuple2<Long, Long>> out) {
						  }
				})
				.map(new IdentityMapper<Tuple2<Long, Long>>());

		DataSet<Tuple2<Long, Long>> joined = sourceB.join(sourceC)
				.where(0).equalTo(1)
				.with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());

		DataSet<Tuple2<Long, Long>> joined2 = mapped.join(joined)
				.where(1).equalTo(1)
				.with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());

		DataSet<Tuple2<Long, Long>> reduced = mapped
				.groupBy(1)
				.reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>());

		reduced.cross(joined2)
				.output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());

		joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
		joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());

		Plan plan = env.createProgramPlan();
		OptimizedPlan oPlan = compileNoStats(plan);
		new JobGraphGenerator().compileJobGraph(oPlan);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #10

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

4 votes

@Test
public void testBranchingBroadcastVariable() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(100);

	DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1");
	DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2");
	DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3");
	
	DataSet<String> result1 = input1
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>())
				.withBroadcastSet(input3, "bc");
	
	DataSet<String> result2 = input2
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>())
				.withBroadcastSet(input3, "bc");
	
	result1.join(result2)
			.where(new IdentityKeyExtractor<String>())
			.equalTo(new IdentityKeyExtractor<String>())
			.with(new RichJoinFunction<String, String, String>() {
				@Override
				public String join(String first, String second) {
					return null;
				}
			})
			.withBroadcastSet(input3, "bc1")
			.withBroadcastSet(input1, "bc2")
			.withBroadcastSet(result1, "bc3")
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #11

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

4 votes

/**
 * 
 * <pre>

 *              (SINK A)
 *                  |    (SINK B)    (SINK C)
 *                CROSS    /          /
 *               /     \   |  +------+
 *              /       \  | /
 *          REDUCE      MATCH2
 *             |    +---/    \
 *              \  /          |
 *               MAP          |
 *                |           |
 *             COGROUP      MATCH1
 *             /     \     /     \
 *        (SRC A)    (SRC B)    (SRC C)
 * </pre>
 */
@Test
public void testBranchingWithMultipleDataSinks() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(DEFAULT_PARALLELISM);

		DataSet<Tuple2<Long, Long>> sourceA = env.generateSequence(1, 10000000)
				.map(new Duplicator<Long>());

		DataSet<Tuple2<Long, Long>> sourceB = env.generateSequence(1, 10000000)
				.map(new Duplicator<Long>());

		DataSet<Tuple2<Long, Long>> sourceC = env.generateSequence(1, 10000000)
				.map(new Duplicator<Long>());

		DataSet<Tuple2<Long, Long>> mapped = sourceA.coGroup(sourceB)
				.where(0).equalTo(1)
				.with(new CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>>() {
						@Override
						public void coGroup(Iterable<Tuple2<Long, Long>> first,
												Iterable<Tuple2<Long, Long>> second,
												Collector<Tuple2<Long, Long>> out) {
						  }
				})
				.map(new IdentityMapper<Tuple2<Long, Long>>());

		DataSet<Tuple2<Long, Long>> joined = sourceB.join(sourceC)
				.where(0).equalTo(1)
				.with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());

		DataSet<Tuple2<Long, Long>> joined2 = mapped.join(joined)
				.where(1).equalTo(1)
				.with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());

		DataSet<Tuple2<Long, Long>> reduced = mapped
				.groupBy(1)
				.reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>());

		reduced.cross(joined2)
				.output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());

		joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
		joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());

		Plan plan = env.createProgramPlan();
		OptimizedPlan oPlan = compileNoStats(plan);
		new JobGraphGenerator().compileJobGraph(oPlan);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #12

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

4 votes

@Test
public void testBranchingBroadcastVariable() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(100);

	DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1");
	DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2");
	DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3");
	
	DataSet<String> result1 = input1
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>())
				.withBroadcastSet(input3, "bc");
	
	DataSet<String> result2 = input2
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>())
				.withBroadcastSet(input3, "bc");
	
	result1.join(result2)
			.where(new IdentityKeyExtractor<String>())
			.equalTo(new IdentityKeyExtractor<String>())
			.with(new RichJoinFunction<String, String, String>() {
				@Override
				public String join(String first, String second) {
					return null;
				}
			})
			.withBroadcastSet(input3, "bc1")
			.withBroadcastSet(input1, "bc2")
			.withBroadcastSet(result1, "bc3")
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #13

Source File: BranchingPlansCompilerTest.java From Flink-CEPplus with Apache License 2.0

3 votes

@Test
public void testBCVariableClosure() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	
	DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
	
	DataSet<String> reduced = input
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>());
	
	
	DataSet<String> initialSolution = input.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc");
	
	
	IterativeDataSet<String> iteration = initialSolution.iterate(100);
	
	iteration.closeWith(iteration.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "red"))
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #14

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

3 votes

@Test
public void testBCVariableClosure() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	
	DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
	
	DataSet<String> reduced = input
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>());
	
	
	DataSet<String> initialSolution = input.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc");
	
	
	IterativeDataSet<String> iteration = initialSolution.iterate(100);
	
	iteration.closeWith(iteration.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "red"))
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #15

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

3 votes

@Test
public void testBCVariableClosure() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	
	DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
	
	DataSet<String> reduced = input
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>());
	
	
	DataSet<String> initialSolution = input.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc");
	
	
	IterativeDataSet<String> iteration = initialSolution.iterate(100);
	
	iteration.closeWith(iteration.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "red"))
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}