Java Code Examples for org.apache.flink.graph.Graph#runScatterGatherIteration()
The following examples show how to use
org.apache.flink.graph.Graph#runScatterGatherIteration() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CollectionModeSuperstepITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testProgram() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); Graph<Long, Long, Long> graph = Graph.fromCollection(TestGraphUtils.getLongLongVertices(), TestGraphUtils.getLongLongEdges(), env).mapVertices(new AssignOneMapper()); Graph<Long, Long, Long> result = graph.runScatterGatherIteration( new MessageFunction(), new UpdateFunction(), 10); result.getVertices().map( new VertexToTuple2Map<>()).output( new DiscardingOutputFormat<>()); env.execute(); }
Example 2
Source File: ScatterGatherConfigurationITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testDefaultConfiguration() throws Exception { /* * Test Graph's runScatterGatherIteration when configuration parameters are not provided * i.e. degrees and numVertices will be -1, EdgeDirection will be OUT. */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); Graph<Long, Long, Long> graph = Graph.fromCollection(TestGraphUtils.getLongLongVertices(), TestGraphUtils.getLongLongEdges(), env).mapVertices(new AssignOneMapper()); Graph<Long, Long, Long> res = graph.runScatterGatherIteration( new MessageFunctionDefault(), new UpdateFunctionDefault(), 5); DataSet<Tuple2<Long, Long>> data = res.getVertices().map(new VertexToTuple2Map<>()); List<Tuple2<Long, Long>> result = data.collect(); expectedResult = "1,6\n" + "2,6\n" + "3,6\n" + "4,6\n" + "5,6"; compareResultAsTuples(result, expectedResult); }
Example 3
Source File: ScatterGatherConfigurationITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testDefaultConfiguration() throws Exception { /* * Test Graph's runScatterGatherIteration when configuration parameters are not provided * i.e. degrees and numVertices will be -1, EdgeDirection will be OUT. */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); Graph<Long, Long, Long> graph = Graph.fromCollection(TestGraphUtils.getLongLongVertices(), TestGraphUtils.getLongLongEdges(), env).mapVertices(new AssignOneMapper()); Graph<Long, Long, Long> res = graph.runScatterGatherIteration( new MessageFunctionDefault(), new UpdateFunctionDefault(), 5); DataSet<Tuple2<Long, Long>> data = res.getVertices().map(new VertexToTuple2Map<>()); List<Tuple2<Long, Long>> result = data.collect(); expectedResult = "1,6\n" + "2,6\n" + "3,6\n" + "4,6\n" + "5,6"; compareResultAsTuples(result, expectedResult); }
Example 4
Source File: CollectionModeSuperstepITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testProgram() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); Graph<Long, Long, Long> graph = Graph.fromCollection(TestGraphUtils.getLongLongVertices(), TestGraphUtils.getLongLongEdges(), env).mapVertices(new AssignOneMapper()); Graph<Long, Long, Long> result = graph.runScatterGatherIteration( new MessageFunction(), new UpdateFunction(), 10); result.getVertices().map( new VertexToTuple2Map<>()).output( new DiscardingOutputFormat<>()); env.execute(); }
Example 5
Source File: CollectionModeSuperstepITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testProgram() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); Graph<Long, Long, Long> graph = Graph.fromCollection(TestGraphUtils.getLongLongVertices(), TestGraphUtils.getLongLongEdges(), env).mapVertices(new AssignOneMapper()); Graph<Long, Long, Long> result = graph.runScatterGatherIteration( new MessageFunction(), new UpdateFunction(), 10); result.getVertices().map( new VertexToTuple2Map<>()).output( new DiscardingOutputFormat<>()); env.execute(); }
Example 6
Source File: ScatterGatherConfigurationITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testDefaultConfiguration() throws Exception { /* * Test Graph's runScatterGatherIteration when configuration parameters are not provided * i.e. degrees and numVertices will be -1, EdgeDirection will be OUT. */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); Graph<Long, Long, Long> graph = Graph.fromCollection(TestGraphUtils.getLongLongVertices(), TestGraphUtils.getLongLongEdges(), env).mapVertices(new AssignOneMapper()); Graph<Long, Long, Long> res = graph.runScatterGatherIteration( new MessageFunctionDefault(), new UpdateFunctionDefault(), 5); DataSet<Tuple2<Long, Long>> data = res.getVertices().map(new VertexToTuple2Map<>()); List<Tuple2<Long, Long>> result = data.collect(); expectedResult = "1,6\n" + "2,6\n" + "3,6\n" + "4,6\n" + "5,6"; compareResultAsTuples(result, expectedResult); }
Example 7
Source File: ScatterGatherConfigurationITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testRunWithConfiguration() throws Exception { /* * Test Graph's runScatterGatherIteration when configuration parameters are provided */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); Graph<Long, Long, Long> graph = Graph.fromCollection(TestGraphUtils.getLongLongVertices(), TestGraphUtils.getLongLongEdges(), env).mapVertices(new AssignOneMapper()); // create the configuration object ScatterGatherConfiguration parameters = new ScatterGatherConfiguration(); parameters.addBroadcastSetForScatterFunction("messagingBcastSet", env.fromElements(4, 5, 6)); parameters.addBroadcastSetForGatherFunction("updateBcastSet", env.fromElements(1, 2, 3)); parameters.registerAggregator("superstepAggregator", new LongSumAggregator()); parameters.setOptNumVertices(true); Graph<Long, Long, Long> res = graph.runScatterGatherIteration( new MessageFunction(), new UpdateFunction(), 10, parameters); DataSet<Vertex<Long, Long>> data = res.getVertices(); List<Vertex<Long, Long>> result = data.collect(); expectedResult = "1,11\n" + "2,11\n" + "3,11\n" + "4,11\n" + "5,11"; compareResultAsTuples(result, expectedResult); }
Example 8
Source File: ScatterGatherConfigurationITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testRunWithConfiguration() throws Exception { /* * Test Graph's runScatterGatherIteration when configuration parameters are provided */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); Graph<Long, Long, Long> graph = Graph.fromCollection(TestGraphUtils.getLongLongVertices(), TestGraphUtils.getLongLongEdges(), env).mapVertices(new AssignOneMapper()); // create the configuration object ScatterGatherConfiguration parameters = new ScatterGatherConfiguration(); parameters.addBroadcastSetForScatterFunction("messagingBcastSet", env.fromElements(4, 5, 6)); parameters.addBroadcastSetForGatherFunction("updateBcastSet", env.fromElements(1, 2, 3)); parameters.registerAggregator("superstepAggregator", new LongSumAggregator()); parameters.setOptNumVertices(true); Graph<Long, Long, Long> res = graph.runScatterGatherIteration( new MessageFunction(), new UpdateFunction(), 10, parameters); DataSet<Vertex<Long, Long>> data = res.getVertices(); List<Vertex<Long, Long>> result = data.collect(); expectedResult = "1,11\n" + "2,11\n" + "3,11\n" + "4,11\n" + "5,11"; compareResultAsTuples(result, expectedResult); }
Example 9
Source File: IncrementalSSSPITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testIncrementalSSSPNonSPEdge() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Vertex<Long, Double>> vertices = IncrementalSSSPData.getDefaultVertexDataSet(env); DataSet<Edge<Long, Double>> edges = IncrementalSSSPData.getDefaultEdgeDataSet(env); DataSet<Edge<Long, Double>> edgesInSSSP = IncrementalSSSPData.getDefaultEdgesInSSSP(env); // the edge to be removed is a non-SP edge Edge<Long, Double> edgeToBeRemoved = new Edge<>(3L, 5L, 5.0); Graph<Long, Double, Double> graph = Graph.fromDataSet(vertices, edges, env); // Assumption: all minimum weight paths are kept Graph<Long, Double, Double> ssspGraph = Graph.fromDataSet(vertices, edgesInSSSP, env); // remove the edge graph.removeEdge(edgeToBeRemoved); // configure the iteration ScatterGatherConfiguration parameters = new ScatterGatherConfiguration(); if (IncrementalSSSP.isInSSSP(edgeToBeRemoved, edgesInSSSP)) { parameters.setDirection(EdgeDirection.IN); parameters.setOptDegrees(true); // run the scatter gather iteration to propagate info Graph<Long, Double, Double> result = ssspGraph.runScatterGatherIteration( new IncrementalSSSP.InvalidateMessenger(edgeToBeRemoved), new IncrementalSSSP.VertexDistanceUpdater(), IncrementalSSSPData.NUM_VERTICES, parameters); DataSet<Vertex<Long, Double>> resultedVertices = result.getVertices(); resultedVertices.writeAsCsv(resultPath, "\n", ","); env.execute(); } else { vertices.writeAsCsv(resultPath, "\n", ","); env.execute(); } expected = IncrementalSSSPData.VERTICES; }
Example 10
Source File: SingleSourceShortestPaths.java From flink with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { if (!parseParameters(args)) { return; } ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Edge<Long, Double>> edges = getEdgesDataSet(env); Graph<Long, Double, Double> graph = Graph.fromDataSet(edges, new InitVertices(srcVertexId), env); // Execute the scatter-gather iteration Graph<Long, Double, Double> result = graph.runScatterGatherIteration( new MinDistanceMessenger(), new VertexDistanceUpdater(), maxIterations); // Extract the vertices as the result DataSet<Vertex<Long, Double>> singleSourceShortestPaths = result.getVertices(); // emit result if (fileOutput) { singleSourceShortestPaths.writeAsCsv(outputPath, "\n", ","); // since file sinks are lazy, we trigger the execution explicitly env.execute("Single Source Shortest Paths Example"); } else { singleSourceShortestPaths.print(); } }
Example 11
Source File: IncrementalSSSPITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testIncrementalSSSPNonSPEdge() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Vertex<Long, Double>> vertices = IncrementalSSSPData.getDefaultVertexDataSet(env); DataSet<Edge<Long, Double>> edges = IncrementalSSSPData.getDefaultEdgeDataSet(env); DataSet<Edge<Long, Double>> edgesInSSSP = IncrementalSSSPData.getDefaultEdgesInSSSP(env); // the edge to be removed is a non-SP edge Edge<Long, Double> edgeToBeRemoved = new Edge<>(3L, 5L, 5.0); Graph<Long, Double, Double> graph = Graph.fromDataSet(vertices, edges, env); // Assumption: all minimum weight paths are kept Graph<Long, Double, Double> ssspGraph = Graph.fromDataSet(vertices, edgesInSSSP, env); // remove the edge graph.removeEdge(edgeToBeRemoved); // configure the iteration ScatterGatherConfiguration parameters = new ScatterGatherConfiguration(); if (IncrementalSSSP.isInSSSP(edgeToBeRemoved, edgesInSSSP)) { parameters.setDirection(EdgeDirection.IN); parameters.setOptDegrees(true); // run the scatter gather iteration to propagate info Graph<Long, Double, Double> result = ssspGraph.runScatterGatherIteration( new IncrementalSSSP.InvalidateMessenger(edgeToBeRemoved), new IncrementalSSSP.VertexDistanceUpdater(), IncrementalSSSPData.NUM_VERTICES, parameters); DataSet<Vertex<Long, Double>> resultedVertices = result.getVertices(); resultedVertices.writeAsCsv(resultPath, "\n", ","); env.execute(); } else { vertices.writeAsCsv(resultPath, "\n", ","); env.execute(); } expected = IncrementalSSSPData.VERTICES; }
Example 12
Source File: SingleSourceShortestPaths.java From flink with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { if (!parseParameters(args)) { return; } ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Edge<Long, Double>> edges = getEdgesDataSet(env); Graph<Long, Double, Double> graph = Graph.fromDataSet(edges, new InitVertices(srcVertexId), env); // Execute the scatter-gather iteration Graph<Long, Double, Double> result = graph.runScatterGatherIteration( new MinDistanceMessenger(), new VertexDistanceUpdater(), maxIterations); // Extract the vertices as the result DataSet<Vertex<Long, Double>> singleSourceShortestPaths = result.getVertices(); // emit result if (fileOutput) { singleSourceShortestPaths.writeAsCsv(outputPath, "\n", ","); // since file sinks are lazy, we trigger the execution explicitly env.execute("Single Source Shortest Paths Example"); } else { singleSourceShortestPaths.print(); } }
Example 13
Source File: ScatterGatherConfigurationITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testRunWithConfiguration() throws Exception { /* * Test Graph's runScatterGatherIteration when configuration parameters are provided */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); Graph<Long, Long, Long> graph = Graph.fromCollection(TestGraphUtils.getLongLongVertices(), TestGraphUtils.getLongLongEdges(), env).mapVertices(new AssignOneMapper()); // create the configuration object ScatterGatherConfiguration parameters = new ScatterGatherConfiguration(); parameters.addBroadcastSetForScatterFunction("messagingBcastSet", env.fromElements(4, 5, 6)); parameters.addBroadcastSetForGatherFunction("updateBcastSet", env.fromElements(1, 2, 3)); parameters.registerAggregator("superstepAggregator", new LongSumAggregator()); parameters.setOptNumVertices(true); Graph<Long, Long, Long> res = graph.runScatterGatherIteration( new MessageFunction(), new UpdateFunction(), 10, parameters); DataSet<Vertex<Long, Long>> data = res.getVertices(); List<Vertex<Long, Long>> result = data.collect(); expectedResult = "1,11\n" + "2,11\n" + "3,11\n" + "4,11\n" + "5,11"; compareResultAsTuples(result, expectedResult); }
Example 14
Source File: IncrementalSSSPITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testIncrementalSSSPNonSPEdge() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Vertex<Long, Double>> vertices = IncrementalSSSPData.getDefaultVertexDataSet(env); DataSet<Edge<Long, Double>> edges = IncrementalSSSPData.getDefaultEdgeDataSet(env); DataSet<Edge<Long, Double>> edgesInSSSP = IncrementalSSSPData.getDefaultEdgesInSSSP(env); // the edge to be removed is a non-SP edge Edge<Long, Double> edgeToBeRemoved = new Edge<>(3L, 5L, 5.0); Graph<Long, Double, Double> graph = Graph.fromDataSet(vertices, edges, env); // Assumption: all minimum weight paths are kept Graph<Long, Double, Double> ssspGraph = Graph.fromDataSet(vertices, edgesInSSSP, env); // remove the edge graph.removeEdge(edgeToBeRemoved); // configure the iteration ScatterGatherConfiguration parameters = new ScatterGatherConfiguration(); if (IncrementalSSSP.isInSSSP(edgeToBeRemoved, edgesInSSSP)) { parameters.setDirection(EdgeDirection.IN); parameters.setOptDegrees(true); // run the scatter gather iteration to propagate info Graph<Long, Double, Double> result = ssspGraph.runScatterGatherIteration( new IncrementalSSSP.InvalidateMessenger(edgeToBeRemoved), new IncrementalSSSP.VertexDistanceUpdater(), IncrementalSSSPData.NUM_VERTICES, parameters); DataSet<Vertex<Long, Double>> resultedVertices = result.getVertices(); resultedVertices.writeAsCsv(resultPath, "\n", ","); env.execute(); } else { vertices.writeAsCsv(resultPath, "\n", ","); env.execute(); } expected = IncrementalSSSPData.VERTICES; }
Example 15
Source File: SingleSourceShortestPaths.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { if (!parseParameters(args)) { return; } ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Edge<Long, Double>> edges = getEdgesDataSet(env); Graph<Long, Double, Double> graph = Graph.fromDataSet(edges, new InitVertices(srcVertexId), env); // Execute the scatter-gather iteration Graph<Long, Double, Double> result = graph.runScatterGatherIteration( new MinDistanceMessenger(), new VertexDistanceUpdater(), maxIterations); // Extract the vertices as the result DataSet<Vertex<Long, Double>> singleSourceShortestPaths = result.getVertices(); // emit result if (fileOutput) { singleSourceShortestPaths.writeAsCsv(outputPath, "\n", ","); // since file sinks are lazy, we trigger the execution explicitly env.execute("Single Source Shortest Paths Example"); } else { singleSourceShortestPaths.print(); } }
Example 16
Source File: IncrementalSSSP.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
public static void main(String [] args) throws Exception { if (!parseParameters(args)) { return; } ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); Edge<Long, Double> edgeToBeRemoved = getEdgeToBeRemoved(); Graph<Long, Double, Double> graph = IncrementalSSSP.getGraph(env); // Assumption: all minimum weight paths are kept Graph<Long, Double, Double> ssspGraph = IncrementalSSSP.getSSSPGraph(env); // remove the edge graph.removeEdge(edgeToBeRemoved); // configure the iteration ScatterGatherConfiguration parameters = new ScatterGatherConfiguration(); if (isInSSSP(edgeToBeRemoved, ssspGraph.getEdges())) { parameters.setDirection(EdgeDirection.IN); parameters.setOptDegrees(true); // run the scatter-gather iteration to propagate info Graph<Long, Double, Double> result = ssspGraph.runScatterGatherIteration(new InvalidateMessenger(edgeToBeRemoved), new VertexDistanceUpdater(), maxIterations, parameters); DataSet<Vertex<Long, Double>> resultedVertices = result.getVertices(); // Emit results if (fileOutput) { resultedVertices.writeAsCsv(outputPath, "\n", ","); env.execute("Incremental SSSP Example"); } else { resultedVertices.print(); } } else { // print the vertices if (fileOutput) { graph.getVertices().writeAsCsv(outputPath, "\n", ","); env.execute("Incremental SSSP Example"); } else { graph.getVertices().print(); } } }
Example 17
Source File: IncrementalSSSP.java From flink with Apache License 2.0 | 4 votes |
public static void main(String [] args) throws Exception { if (!parseParameters(args)) { return; } ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); Edge<Long, Double> edgeToBeRemoved = getEdgeToBeRemoved(); Graph<Long, Double, Double> graph = IncrementalSSSP.getGraph(env); // Assumption: all minimum weight paths are kept Graph<Long, Double, Double> ssspGraph = IncrementalSSSP.getSSSPGraph(env); // remove the edge graph.removeEdge(edgeToBeRemoved); // configure the iteration ScatterGatherConfiguration parameters = new ScatterGatherConfiguration(); if (isInSSSP(edgeToBeRemoved, ssspGraph.getEdges())) { parameters.setDirection(EdgeDirection.IN); parameters.setOptDegrees(true); // run the scatter-gather iteration to propagate info Graph<Long, Double, Double> result = ssspGraph.runScatterGatherIteration(new InvalidateMessenger(edgeToBeRemoved), new VertexDistanceUpdater(), maxIterations, parameters); DataSet<Vertex<Long, Double>> resultedVertices = result.getVertices(); // Emit results if (fileOutput) { resultedVertices.writeAsCsv(outputPath, "\n", ","); env.execute("Incremental SSSP Example"); } else { resultedVertices.print(); } } else { // print the vertices if (fileOutput) { graph.getVertices().writeAsCsv(outputPath, "\n", ","); env.execute("Incremental SSSP Example"); } else { graph.getVertices().print(); } } }
Example 18
Source File: IncrementalSSSP.java From flink with Apache License 2.0 | 4 votes |
public static void main(String [] args) throws Exception { if (!parseParameters(args)) { return; } ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); Edge<Long, Double> edgeToBeRemoved = getEdgeToBeRemoved(); Graph<Long, Double, Double> graph = IncrementalSSSP.getGraph(env); // Assumption: all minimum weight paths are kept Graph<Long, Double, Double> ssspGraph = IncrementalSSSP.getSSSPGraph(env); // remove the edge graph.removeEdge(edgeToBeRemoved); // configure the iteration ScatterGatherConfiguration parameters = new ScatterGatherConfiguration(); if (isInSSSP(edgeToBeRemoved, ssspGraph.getEdges())) { parameters.setDirection(EdgeDirection.IN); parameters.setOptDegrees(true); // run the scatter-gather iteration to propagate info Graph<Long, Double, Double> result = ssspGraph.runScatterGatherIteration(new InvalidateMessenger(edgeToBeRemoved), new VertexDistanceUpdater(), maxIterations, parameters); DataSet<Vertex<Long, Double>> resultedVertices = result.getVertices(); // Emit results if (fileOutput) { resultedVertices.writeAsCsv(outputPath, "\n", ","); env.execute("Incremental SSSP Example"); } else { resultedVertices.print(); } } else { // print the vertices if (fileOutput) { graph.getVertices().writeAsCsv(outputPath, "\n", ","); env.execute("Incremental SSSP Example"); } else { graph.getVertices().print(); } } }