Java Code Examples for org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator#getOptimizerPlanAsJSON()
The following examples show how to use
org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator#getOptimizerPlanAsJSON() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ClientTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testGetExecutionPlan() throws ProgramInvocationException { PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp"); assertNotNull(prg.getPreviewPlan()); Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config); OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, 1); assertNotNull(op); PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); assertNotNull(dumper.getOptimizerPlanAsJSON(op)); // test HTML escaping PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator(); dumper2.setEncodeForHTML(true); String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op); assertEquals(-1, htmlEscaped.indexOf('\\')); }
Example 2
Source File: ClientTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testGetExecutionPlan() throws ProgramInvocationException { PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp"); assertNotNull(prg.getPreviewPlan()); Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config); OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, 1); assertNotNull(op); PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); assertNotNull(dumper.getOptimizerPlanAsJSON(op)); // test HTML escaping PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator(); dumper2.setEncodeForHTML(true); String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op); assertEquals(-1, htmlEscaped.indexOf('\\')); }
Example 3
Source File: DumpCompiledPlanTest.java From flink with Apache License 2.0 | 6 votes |
private void verifyOptimizedPlan(Class<?> entrypoint, String... args) throws Exception { final PackagedProgram program = PackagedProgram .newBuilder() .setEntryPointClassName(entrypoint.getName()) .setArguments(args) .build(); final Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(program, new Configuration(), 1, true); assertTrue(pipeline instanceof Plan); final Plan plan = (Plan) pipeline; final OptimizedPlan op = compileNoStats(plan); final PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); final String json = dumper.getOptimizerPlanAsJSON(op); try (JsonParser parser = new JsonFactory().createParser(json)) { while (parser.nextToken() != null) { } } }
Example 4
Source File: ClientTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testGetExecutionPlan() throws ProgramInvocationException { PackagedProgram prg = PackagedProgram.newBuilder() .setEntryPointClassName(TestOptimizerPlan.class.getName()) .setArguments("/dev/random", "/tmp") .build(); Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config); Plan plan = (Plan) PackagedProgramUtils.getPipelineFromProgram(prg, new Configuration(), 1, true); OptimizedPlan op = optimizer.compile(plan); assertNotNull(op); PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); assertNotNull(dumper.getOptimizerPlanAsJSON(op)); // test HTML escaping PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator(); dumper2.setEncodeForHTML(true); String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op); assertEquals(-1, htmlEscaped.indexOf('\\')); }
Example 5
Source File: TestEnvironment.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public String getExecutionPlan() throws Exception { OptimizedPlan op = compileProgram("unused"); PlanJSONDumpGenerator jsonGen = new PlanJSONDumpGenerator(); return jsonGen.getOptimizerPlanAsJSON(op); }
Example 6
Source File: ContextEnvironment.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public String getExecutionPlan() throws Exception { Plan plan = createProgramPlan("unnamed job"); OptimizedPlan op = ClusterClient.getOptimizedPlan(client.compiler, plan, getParallelism()); PlanJSONDumpGenerator gen = new PlanJSONDumpGenerator(); return gen.getOptimizerPlanAsJSON(op); }
Example 7
Source File: ExecutionPlanCreationTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testGetExecutionPlan() { try { PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp"); assertNotNull(prg.getPreviewPlan()); InetAddress mockAddress = InetAddress.getLocalHost(); InetSocketAddress mockJmAddress = new InetSocketAddress(mockAddress, 12345); Configuration config = new Configuration(); config.setString(JobManagerOptions.ADDRESS, mockJmAddress.getHostName()); config.setInteger(JobManagerOptions.PORT, mockJmAddress.getPort()); Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config); OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, -1); assertNotNull(op); PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); assertNotNull(dumper.getOptimizerPlanAsJSON(op)); // test HTML escaping PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator(); dumper2.setEncodeForHTML(true); String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op); assertEquals(-1, htmlEscaped.indexOf('\\')); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
Example 8
Source File: TestEnvironment.java From flink with Apache License 2.0 | 5 votes |
@Override public String getExecutionPlan() throws Exception { OptimizedPlan op = compileProgram("unused"); PlanJSONDumpGenerator jsonGen = new PlanJSONDumpGenerator(); return jsonGen.getOptimizerPlanAsJSON(op); }
Example 9
Source File: ContextEnvironment.java From flink with Apache License 2.0 | 5 votes |
@Override public String getExecutionPlan() throws Exception { Plan plan = createProgramPlan("unnamed job"); OptimizedPlan op = ClusterClient.getOptimizedPlan(client.compiler, plan, getParallelism()); PlanJSONDumpGenerator gen = new PlanJSONDumpGenerator(); return gen.getOptimizerPlanAsJSON(op); }
Example 10
Source File: ExecutionPlanCreationTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGetExecutionPlan() { try { PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp"); assertNotNull(prg.getPreviewPlan()); InetAddress mockAddress = InetAddress.getLocalHost(); InetSocketAddress mockJmAddress = new InetSocketAddress(mockAddress, 12345); Configuration config = new Configuration(); config.setString(JobManagerOptions.ADDRESS, mockJmAddress.getHostName()); config.setInteger(JobManagerOptions.PORT, mockJmAddress.getPort()); Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config); OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, -1); assertNotNull(op); PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); assertNotNull(dumper.getOptimizerPlanAsJSON(op)); // test HTML escaping PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator(); dumper2.setEncodeForHTML(true); String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op); assertEquals(-1, htmlEscaped.indexOf('\\')); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
Example 11
Source File: ExecutionPlanCreationTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGetExecutionPlan() { try { PackagedProgram prg = PackagedProgram.newBuilder() .setEntryPointClassName(TestOptimizerPlan.class.getName()) .setArguments("/dev/random", "/tmp") .build(); InetAddress mockAddress = InetAddress.getLocalHost(); InetSocketAddress mockJmAddress = new InetSocketAddress(mockAddress, 12345); Configuration config = new Configuration(); config.setString(JobManagerOptions.ADDRESS, mockJmAddress.getHostName()); config.setInteger(JobManagerOptions.PORT, mockJmAddress.getPort()); Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config); Plan plan = (Plan) PackagedProgramUtils.getPipelineFromProgram(prg, config, -1, true); OptimizedPlan op = optimizer.compile(plan); assertNotNull(op); PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); assertNotNull(dumper.getOptimizerPlanAsJSON(op)); // test HTML escaping PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator(); dumper2.setEncodeForHTML(true); String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op); assertEquals(-1, htmlEscaped.indexOf('\\')); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
Example 12
Source File: ConnectedComponentsCoGroupTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testWorksetConnectedComponents() { Plan plan = getConnectedComponentsCoGroupPlan(); plan.setExecutionConfig(new ExecutionConfig()); OptimizedPlan optPlan = compileNoStats(plan); OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(optPlan); if (PRINT_PLAN) { PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); String json = dumper.getOptimizerPlanAsJSON(optPlan); System.out.println(json); } SourcePlanNode vertexSource = or.getNode(VERTEX_SOURCE); SourcePlanNode edgesSource = or.getNode(EDGES_SOURCE); SinkPlanNode sink = or.getNode(SINK); WorksetIterationPlanNode iter = or.getNode(ITERATION_NAME); DualInputPlanNode neighborsJoin = or.getNode(JOIN_NEIGHBORS_MATCH); DualInputPlanNode cogroup = or.getNode(MIN_ID_AND_UPDATE); // -------------------------------------------------------------------- // Plan validation: // // We expect the plan to go with a sort-merge join, because the CoGroup // sorts and the join in the successive iteration can re-exploit the sorting. // -------------------------------------------------------------------- // test all drivers Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy()); Assert.assertEquals(DriverStrategy.NONE, vertexSource.getDriverStrategy()); Assert.assertEquals(DriverStrategy.NONE, edgesSource.getDriverStrategy()); Assert.assertEquals(DriverStrategy.INNER_MERGE, neighborsJoin.getDriverStrategy()); Assert.assertEquals(set0, neighborsJoin.getKeysForInput1()); Assert.assertEquals(set0, neighborsJoin.getKeysForInput2()); Assert.assertEquals(DriverStrategy.CO_GROUP, cogroup.getDriverStrategy()); Assert.assertEquals(set0, cogroup.getKeysForInput1()); Assert.assertEquals(set0, cogroup.getKeysForInput2()); // test all the shipping strategies Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialSolutionSetInput().getShipStrategy()); Assert.assertEquals(set0, iter.getInitialSolutionSetInput().getShipStrategyKeys()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialWorksetInput().getShipStrategy()); Assert.assertEquals(set0, iter.getInitialWorksetInput().getShipStrategyKeys()); Assert.assertEquals(ShipStrategyType.FORWARD, neighborsJoin.getInput1().getShipStrategy()); // workset Assert.assertEquals(ShipStrategyType.PARTITION_HASH, neighborsJoin.getInput2().getShipStrategy()); // edges Assert.assertEquals(set0, neighborsJoin.getInput2().getShipStrategyKeys()); Assert.assertTrue(neighborsJoin.getInput2().getTempMode().isCached()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, cogroup.getInput1().getShipStrategy()); // min id Assert.assertEquals(ShipStrategyType.FORWARD, cogroup.getInput2().getShipStrategy()); // solution set // test all the local strategies Assert.assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, iter.getInitialSolutionSetInput().getLocalStrategy()); // the sort for the neighbor join in the first iteration is pushed out of the loop Assert.assertEquals(LocalStrategy.SORT, iter.getInitialWorksetInput().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, neighborsJoin.getInput1().getLocalStrategy()); // workset Assert.assertEquals(LocalStrategy.SORT, neighborsJoin.getInput2().getLocalStrategy()); // edges Assert.assertEquals(LocalStrategy.SORT, cogroup.getInput1().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, cogroup.getInput2().getLocalStrategy()); // solution set // check the caches Assert.assertTrue(TempMode.CACHED == neighborsJoin.getInput2().getTempMode()); JobGraphGenerator jgg = new JobGraphGenerator(); jgg.compileJobGraph(optPlan); }
Example 13
Source File: ClusterClient.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
public static String getOptimizedPlanAsJson(Optimizer compiler, PackagedProgram prog, int parallelism) throws CompilerException, ProgramInvocationException { PlanJSONDumpGenerator jsonGen = new PlanJSONDumpGenerator(); return jsonGen.getOptimizerPlanAsJSON((OptimizedPlan) getOptimizedPlan(compiler, prog, parallelism)); }
Example 14
Source File: ConnectedComponentsCoGroupTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testWorksetConnectedComponents() { Plan plan = getConnectedComponentsCoGroupPlan(); plan.setExecutionConfig(new ExecutionConfig()); OptimizedPlan optPlan = compileNoStats(plan); OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(optPlan); if (PRINT_PLAN) { PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); String json = dumper.getOptimizerPlanAsJSON(optPlan); System.out.println(json); } SourcePlanNode vertexSource = or.getNode(VERTEX_SOURCE); SourcePlanNode edgesSource = or.getNode(EDGES_SOURCE); SinkPlanNode sink = or.getNode(SINK); WorksetIterationPlanNode iter = or.getNode(ITERATION_NAME); DualInputPlanNode neighborsJoin = or.getNode(JOIN_NEIGHBORS_MATCH); DualInputPlanNode cogroup = or.getNode(MIN_ID_AND_UPDATE); // -------------------------------------------------------------------- // Plan validation: // // We expect the plan to go with a sort-merge join, because the CoGroup // sorts and the join in the successive iteration can re-exploit the sorting. // -------------------------------------------------------------------- // test all drivers Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy()); Assert.assertEquals(DriverStrategy.NONE, vertexSource.getDriverStrategy()); Assert.assertEquals(DriverStrategy.NONE, edgesSource.getDriverStrategy()); Assert.assertEquals(DriverStrategy.INNER_MERGE, neighborsJoin.getDriverStrategy()); Assert.assertEquals(set0, neighborsJoin.getKeysForInput1()); Assert.assertEquals(set0, neighborsJoin.getKeysForInput2()); Assert.assertEquals(DriverStrategy.CO_GROUP, cogroup.getDriverStrategy()); Assert.assertEquals(set0, cogroup.getKeysForInput1()); Assert.assertEquals(set0, cogroup.getKeysForInput2()); // test all the shipping strategies Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialSolutionSetInput().getShipStrategy()); Assert.assertEquals(set0, iter.getInitialSolutionSetInput().getShipStrategyKeys()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialWorksetInput().getShipStrategy()); Assert.assertEquals(set0, iter.getInitialWorksetInput().getShipStrategyKeys()); Assert.assertEquals(ShipStrategyType.FORWARD, neighborsJoin.getInput1().getShipStrategy()); // workset Assert.assertEquals(ShipStrategyType.PARTITION_HASH, neighborsJoin.getInput2().getShipStrategy()); // edges Assert.assertEquals(set0, neighborsJoin.getInput2().getShipStrategyKeys()); Assert.assertTrue(neighborsJoin.getInput2().getTempMode().isCached()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, cogroup.getInput1().getShipStrategy()); // min id Assert.assertEquals(ShipStrategyType.FORWARD, cogroup.getInput2().getShipStrategy()); // solution set // test all the local strategies Assert.assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, iter.getInitialSolutionSetInput().getLocalStrategy()); // the sort for the neighbor join in the first iteration is pushed out of the loop Assert.assertEquals(LocalStrategy.SORT, iter.getInitialWorksetInput().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, neighborsJoin.getInput1().getLocalStrategy()); // workset Assert.assertEquals(LocalStrategy.SORT, neighborsJoin.getInput2().getLocalStrategy()); // edges Assert.assertEquals(LocalStrategy.SORT, cogroup.getInput1().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, cogroup.getInput2().getLocalStrategy()); // solution set // check the caches Assert.assertTrue(TempMode.CACHED == neighborsJoin.getInput2().getTempMode()); JobGraphGenerator jgg = new JobGraphGenerator(); jgg.compileJobGraph(optPlan); }
Example 15
Source File: ClusterClient.java From flink with Apache License 2.0 | 4 votes |
public static String getOptimizedPlanAsJson(Optimizer compiler, PackagedProgram prog, int parallelism) throws CompilerException, ProgramInvocationException { PlanJSONDumpGenerator jsonGen = new PlanJSONDumpGenerator(); return jsonGen.getOptimizerPlanAsJSON((OptimizedPlan) getOptimizedPlan(compiler, prog, parallelism)); }
Example 16
Source File: ConnectedComponentsCoGroupTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testWorksetConnectedComponents() throws Exception { Plan plan = getConnectedComponentsCoGroupPlan(); plan.setExecutionConfig(new ExecutionConfig()); OptimizedPlan optPlan = compileNoStats(plan); OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(optPlan); if (PRINT_PLAN) { PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); String json = dumper.getOptimizerPlanAsJSON(optPlan); System.out.println(json); } SourcePlanNode vertexSource = or.getNode(VERTEX_SOURCE); SourcePlanNode edgesSource = or.getNode(EDGES_SOURCE); SinkPlanNode sink = or.getNode(SINK); WorksetIterationPlanNode iter = or.getNode(ITERATION_NAME); DualInputPlanNode neighborsJoin = or.getNode(JOIN_NEIGHBORS_MATCH); DualInputPlanNode cogroup = or.getNode(MIN_ID_AND_UPDATE); // -------------------------------------------------------------------- // Plan validation: // // We expect the plan to go with a sort-merge join, because the CoGroup // sorts and the join in the successive iteration can re-exploit the sorting. // -------------------------------------------------------------------- // test all drivers Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy()); Assert.assertEquals(DriverStrategy.NONE, vertexSource.getDriverStrategy()); Assert.assertEquals(DriverStrategy.NONE, edgesSource.getDriverStrategy()); Assert.assertEquals(DriverStrategy.INNER_MERGE, neighborsJoin.getDriverStrategy()); Assert.assertEquals(set0, neighborsJoin.getKeysForInput1()); Assert.assertEquals(set0, neighborsJoin.getKeysForInput2()); Assert.assertEquals(DriverStrategy.CO_GROUP, cogroup.getDriverStrategy()); Assert.assertEquals(set0, cogroup.getKeysForInput1()); Assert.assertEquals(set0, cogroup.getKeysForInput2()); // test all the shipping strategies Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialSolutionSetInput().getShipStrategy()); Assert.assertEquals(set0, iter.getInitialSolutionSetInput().getShipStrategyKeys()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialWorksetInput().getShipStrategy()); Assert.assertEquals(set0, iter.getInitialWorksetInput().getShipStrategyKeys()); Assert.assertEquals(ShipStrategyType.FORWARD, neighborsJoin.getInput1().getShipStrategy()); // workset Assert.assertEquals(ShipStrategyType.PARTITION_HASH, neighborsJoin.getInput2().getShipStrategy()); // edges Assert.assertEquals(set0, neighborsJoin.getInput2().getShipStrategyKeys()); Assert.assertTrue(neighborsJoin.getInput2().getTempMode().isCached()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, cogroup.getInput1().getShipStrategy()); // min id Assert.assertEquals(ShipStrategyType.FORWARD, cogroup.getInput2().getShipStrategy()); // solution set // test all the local strategies Assert.assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, iter.getInitialSolutionSetInput().getLocalStrategy()); // the sort for the neighbor join in the first iteration is pushed out of the loop Assert.assertEquals(LocalStrategy.SORT, iter.getInitialWorksetInput().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, neighborsJoin.getInput1().getLocalStrategy()); // workset Assert.assertEquals(LocalStrategy.SORT, neighborsJoin.getInput2().getLocalStrategy()); // edges Assert.assertEquals(LocalStrategy.SORT, cogroup.getInput1().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, cogroup.getInput2().getLocalStrategy()); // solution set // check the caches Assert.assertTrue(TempMode.CACHED == neighborsJoin.getInput2().getTempMode()); JobGraphGenerator jgg = new JobGraphGenerator(); jgg.compileJobGraph(optPlan); }