org.apache.flink.optimizer.testfunctions.Top1GroupReducer Java Examples
The following examples show how to use
org.apache.flink.optimizer.testfunctions.Top1GroupReducer.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BranchingPlansCompilerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testMultipleIterations() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(100); DataSet<String> input = env.readTextFile(IN_FILE).name("source1"); DataSet<String> reduced = input .map(new IdentityMapper<String>()) .reduceGroup(new Top1GroupReducer<String>()); IterativeDataSet<String> iteration1 = input.iterate(100); IterativeDataSet<String> iteration2 = input.iterate(20); IterativeDataSet<String> iteration3 = input.iterate(17); iteration1.closeWith(iteration1.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc1")) .output(new DiscardingOutputFormat<String>()); iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(reduced, "bc2")) .output(new DiscardingOutputFormat<String>()); iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()).withBroadcastSet(reduced, "bc3")) .output(new DiscardingOutputFormat<String>()); Plan plan = env.createProgramPlan(); try{ compileNoStats(plan); }catch(Exception e){ e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #2
Source File: BranchingPlansCompilerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testMultipleIterationsWithClosueBCVars() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(100); DataSet<String> input = env.readTextFile(IN_FILE).name("source1"); IterativeDataSet<String> iteration1 = input.iterate(100); IterativeDataSet<String> iteration2 = input.iterate(20); IterativeDataSet<String> iteration3 = input.iterate(17); iteration1.closeWith(iteration1.map(new IdentityMapper<String>())) .output(new DiscardingOutputFormat<String>()); iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>())) .output(new DiscardingOutputFormat<String>()); iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>())) .output(new DiscardingOutputFormat<String>()); Plan plan = env.createProgramPlan(); try{ compileNoStats(plan); }catch(Exception e){ e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #3
Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testMultipleIterations() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(100); DataSet<String> input = env.readTextFile(IN_FILE).name("source1"); DataSet<String> reduced = input .map(new IdentityMapper<String>()) .reduceGroup(new Top1GroupReducer<String>()); IterativeDataSet<String> iteration1 = input.iterate(100); IterativeDataSet<String> iteration2 = input.iterate(20); IterativeDataSet<String> iteration3 = input.iterate(17); iteration1.closeWith(iteration1.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc1")) .output(new DiscardingOutputFormat<String>()); iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(reduced, "bc2")) .output(new DiscardingOutputFormat<String>()); iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()).withBroadcastSet(reduced, "bc3")) .output(new DiscardingOutputFormat<String>()); Plan plan = env.createProgramPlan(); try{ compileNoStats(plan); }catch(Exception e){ e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #4
Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testMultipleIterationsWithClosueBCVars() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(100); DataSet<String> input = env.readTextFile(IN_FILE).name("source1"); IterativeDataSet<String> iteration1 = input.iterate(100); IterativeDataSet<String> iteration2 = input.iterate(20); IterativeDataSet<String> iteration3 = input.iterate(17); iteration1.closeWith(iteration1.map(new IdentityMapper<String>())) .output(new DiscardingOutputFormat<String>()); iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>())) .output(new DiscardingOutputFormat<String>()); iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>())) .output(new DiscardingOutputFormat<String>()); Plan plan = env.createProgramPlan(); try{ compileNoStats(plan); }catch(Exception e){ e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #5
Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testMultipleIterations() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(100); DataSet<String> input = env.readTextFile(IN_FILE).name("source1"); DataSet<String> reduced = input .map(new IdentityMapper<String>()) .reduceGroup(new Top1GroupReducer<String>()); IterativeDataSet<String> iteration1 = input.iterate(100); IterativeDataSet<String> iteration2 = input.iterate(20); IterativeDataSet<String> iteration3 = input.iterate(17); iteration1.closeWith(iteration1.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc1")) .output(new DiscardingOutputFormat<String>()); iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(reduced, "bc2")) .output(new DiscardingOutputFormat<String>()); iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()).withBroadcastSet(reduced, "bc3")) .output(new DiscardingOutputFormat<String>()); Plan plan = env.createProgramPlan(); try{ compileNoStats(plan); }catch(Exception e){ e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #6
Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testMultipleIterationsWithClosueBCVars() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(100); DataSet<String> input = env.readTextFile(IN_FILE).name("source1"); IterativeDataSet<String> iteration1 = input.iterate(100); IterativeDataSet<String> iteration2 = input.iterate(20); IterativeDataSet<String> iteration3 = input.iterate(17); iteration1.closeWith(iteration1.map(new IdentityMapper<String>())) .output(new DiscardingOutputFormat<String>()); iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>())) .output(new DiscardingOutputFormat<String>()); iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>())) .output(new DiscardingOutputFormat<String>()); Plan plan = env.createProgramPlan(); try{ compileNoStats(plan); }catch(Exception e){ e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #7
Source File: BranchingPlansCompilerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * * <pre> * (SINK A) * | (SINK B) (SINK C) * CROSS / / * / \ | +------+ * / \ | / * REDUCE MATCH2 * | +---/ \ * \ / | * MAP | * | | * COGROUP MATCH1 * / \ / \ * (SRC A) (SRC B) (SRC C) * </pre> */ @Test public void testBranchingWithMultipleDataSinks() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(DEFAULT_PARALLELISM); DataSet<Tuple2<Long, Long>> sourceA = env.generateSequence(1, 10000000) .map(new Duplicator<Long>()); DataSet<Tuple2<Long, Long>> sourceB = env.generateSequence(1, 10000000) .map(new Duplicator<Long>()); DataSet<Tuple2<Long, Long>> sourceC = env.generateSequence(1, 10000000) .map(new Duplicator<Long>()); DataSet<Tuple2<Long, Long>> mapped = sourceA.coGroup(sourceB) .where(0).equalTo(1) .with(new CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>>() { @Override public void coGroup(Iterable<Tuple2<Long, Long>> first, Iterable<Tuple2<Long, Long>> second, Collector<Tuple2<Long, Long>> out) { } }) .map(new IdentityMapper<Tuple2<Long, Long>>()); DataSet<Tuple2<Long, Long>> joined = sourceB.join(sourceC) .where(0).equalTo(1) .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>()); DataSet<Tuple2<Long, Long>> joined2 = mapped.join(joined) .where(1).equalTo(1) .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>()); DataSet<Tuple2<Long, Long>> reduced = mapped .groupBy(1) .reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>()); reduced.cross(joined2) .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>()); joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); Plan plan = env.createProgramPlan(); OptimizedPlan oPlan = compileNoStats(plan); new JobGraphGenerator().compileJobGraph(oPlan); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
Example #8
Source File: BranchingPlansCompilerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testBranchingBroadcastVariable() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(100); DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1"); DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2"); DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3"); DataSet<String> result1 = input1 .map(new IdentityMapper<String>()) .reduceGroup(new Top1GroupReducer<String>()) .withBroadcastSet(input3, "bc"); DataSet<String> result2 = input2 .map(new IdentityMapper<String>()) .reduceGroup(new Top1GroupReducer<String>()) .withBroadcastSet(input3, "bc"); result1.join(result2) .where(new IdentityKeyExtractor<String>()) .equalTo(new IdentityKeyExtractor<String>()) .with(new RichJoinFunction<String, String, String>() { @Override public String join(String first, String second) { return null; } }) .withBroadcastSet(input3, "bc1") .withBroadcastSet(input1, "bc2") .withBroadcastSet(result1, "bc3") .output(new DiscardingOutputFormat<String>()); Plan plan = env.createProgramPlan(); try{ compileNoStats(plan); }catch(Exception e){ e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #9
Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0 | 4 votes |
/** * * <pre> * (SINK A) * | (SINK B) (SINK C) * CROSS / / * / \ | +------+ * / \ | / * REDUCE MATCH2 * | +---/ \ * \ / | * MAP | * | | * COGROUP MATCH1 * / \ / \ * (SRC A) (SRC B) (SRC C) * </pre> */ @Test public void testBranchingWithMultipleDataSinks() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(DEFAULT_PARALLELISM); DataSet<Tuple2<Long, Long>> sourceA = env.generateSequence(1, 10000000) .map(new Duplicator<Long>()); DataSet<Tuple2<Long, Long>> sourceB = env.generateSequence(1, 10000000) .map(new Duplicator<Long>()); DataSet<Tuple2<Long, Long>> sourceC = env.generateSequence(1, 10000000) .map(new Duplicator<Long>()); DataSet<Tuple2<Long, Long>> mapped = sourceA.coGroup(sourceB) .where(0).equalTo(1) .with(new CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>>() { @Override public void coGroup(Iterable<Tuple2<Long, Long>> first, Iterable<Tuple2<Long, Long>> second, Collector<Tuple2<Long, Long>> out) { } }) .map(new IdentityMapper<Tuple2<Long, Long>>()); DataSet<Tuple2<Long, Long>> joined = sourceB.join(sourceC) .where(0).equalTo(1) .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>()); DataSet<Tuple2<Long, Long>> joined2 = mapped.join(joined) .where(1).equalTo(1) .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>()); DataSet<Tuple2<Long, Long>> reduced = mapped .groupBy(1) .reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>()); reduced.cross(joined2) .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>()); joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); Plan plan = env.createProgramPlan(); OptimizedPlan oPlan = compileNoStats(plan); new JobGraphGenerator().compileJobGraph(oPlan); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
Example #10
Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testBranchingBroadcastVariable() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(100); DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1"); DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2"); DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3"); DataSet<String> result1 = input1 .map(new IdentityMapper<String>()) .reduceGroup(new Top1GroupReducer<String>()) .withBroadcastSet(input3, "bc"); DataSet<String> result2 = input2 .map(new IdentityMapper<String>()) .reduceGroup(new Top1GroupReducer<String>()) .withBroadcastSet(input3, "bc"); result1.join(result2) .where(new IdentityKeyExtractor<String>()) .equalTo(new IdentityKeyExtractor<String>()) .with(new RichJoinFunction<String, String, String>() { @Override public String join(String first, String second) { return null; } }) .withBroadcastSet(input3, "bc1") .withBroadcastSet(input1, "bc2") .withBroadcastSet(result1, "bc3") .output(new DiscardingOutputFormat<String>()); Plan plan = env.createProgramPlan(); try{ compileNoStats(plan); }catch(Exception e){ e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #11
Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0 | 4 votes |
/** * * <pre> * (SINK A) * | (SINK B) (SINK C) * CROSS / / * / \ | +------+ * / \ | / * REDUCE MATCH2 * | +---/ \ * \ / | * MAP | * | | * COGROUP MATCH1 * / \ / \ * (SRC A) (SRC B) (SRC C) * </pre> */ @Test public void testBranchingWithMultipleDataSinks() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(DEFAULT_PARALLELISM); DataSet<Tuple2<Long, Long>> sourceA = env.generateSequence(1, 10000000) .map(new Duplicator<Long>()); DataSet<Tuple2<Long, Long>> sourceB = env.generateSequence(1, 10000000) .map(new Duplicator<Long>()); DataSet<Tuple2<Long, Long>> sourceC = env.generateSequence(1, 10000000) .map(new Duplicator<Long>()); DataSet<Tuple2<Long, Long>> mapped = sourceA.coGroup(sourceB) .where(0).equalTo(1) .with(new CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>>() { @Override public void coGroup(Iterable<Tuple2<Long, Long>> first, Iterable<Tuple2<Long, Long>> second, Collector<Tuple2<Long, Long>> out) { } }) .map(new IdentityMapper<Tuple2<Long, Long>>()); DataSet<Tuple2<Long, Long>> joined = sourceB.join(sourceC) .where(0).equalTo(1) .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>()); DataSet<Tuple2<Long, Long>> joined2 = mapped.join(joined) .where(1).equalTo(1) .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>()); DataSet<Tuple2<Long, Long>> reduced = mapped .groupBy(1) .reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>()); reduced.cross(joined2) .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>()); joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); Plan plan = env.createProgramPlan(); OptimizedPlan oPlan = compileNoStats(plan); new JobGraphGenerator().compileJobGraph(oPlan); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
Example #12
Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testBranchingBroadcastVariable() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(100); DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1"); DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2"); DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3"); DataSet<String> result1 = input1 .map(new IdentityMapper<String>()) .reduceGroup(new Top1GroupReducer<String>()) .withBroadcastSet(input3, "bc"); DataSet<String> result2 = input2 .map(new IdentityMapper<String>()) .reduceGroup(new Top1GroupReducer<String>()) .withBroadcastSet(input3, "bc"); result1.join(result2) .where(new IdentityKeyExtractor<String>()) .equalTo(new IdentityKeyExtractor<String>()) .with(new RichJoinFunction<String, String, String>() { @Override public String join(String first, String second) { return null; } }) .withBroadcastSet(input3, "bc1") .withBroadcastSet(input1, "bc2") .withBroadcastSet(result1, "bc3") .output(new DiscardingOutputFormat<String>()); Plan plan = env.createProgramPlan(); try{ compileNoStats(plan); }catch(Exception e){ e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #13
Source File: BranchingPlansCompilerTest.java From Flink-CEPplus with Apache License 2.0 | 3 votes |
@Test public void testBCVariableClosure() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<String> input = env.readTextFile(IN_FILE).name("source1"); DataSet<String> reduced = input .map(new IdentityMapper<String>()) .reduceGroup(new Top1GroupReducer<String>()); DataSet<String> initialSolution = input.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc"); IterativeDataSet<String> iteration = initialSolution.iterate(100); iteration.closeWith(iteration.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "red")) .output(new DiscardingOutputFormat<String>()); Plan plan = env.createProgramPlan(); try{ compileNoStats(plan); }catch(Exception e){ e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #14
Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0 | 3 votes |
@Test public void testBCVariableClosure() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<String> input = env.readTextFile(IN_FILE).name("source1"); DataSet<String> reduced = input .map(new IdentityMapper<String>()) .reduceGroup(new Top1GroupReducer<String>()); DataSet<String> initialSolution = input.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc"); IterativeDataSet<String> iteration = initialSolution.iterate(100); iteration.closeWith(iteration.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "red")) .output(new DiscardingOutputFormat<String>()); Plan plan = env.createProgramPlan(); try{ compileNoStats(plan); }catch(Exception e){ e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #15
Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0 | 3 votes |
@Test public void testBCVariableClosure() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<String> input = env.readTextFile(IN_FILE).name("source1"); DataSet<String> reduced = input .map(new IdentityMapper<String>()) .reduceGroup(new Top1GroupReducer<String>()); DataSet<String> initialSolution = input.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc"); IterativeDataSet<String> iteration = initialSolution.iterate(100); iteration.closeWith(iteration.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "red")) .output(new DiscardingOutputFormat<String>()); Plan plan = env.createProgramPlan(); try{ compileNoStats(plan); }catch(Exception e){ e.printStackTrace(); Assert.fail(e.getMessage()); } }