Java Code Examples for org.apache.flink.graph.Graph#fromDataSet()

@Test
public void testCreateWithMapper() throws Exception {
	/*
	 * Test create() with edge dataset and a mapper that assigns the id as value
     */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongEdgeData(env),
		new AssignIdAsValueMapper(), env);

	DataSet<Vertex<Long, Long>> data = graph.getVertices();
	List<Vertex<Long, Long>> result = data.collect();

	expectedResult = "1,1\n" +
		"2,2\n" +
		"3,3\n" +
		"4,4\n" +
		"5,5\n";

	compareResultAsTuples(result, expectedResult);
}
 

@Test
public void testCreateWithoutVertexValues() throws Exception {
	/*
	 * Test create() with edge dataset and no vertex values
     */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	Graph<Long, NullValue, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongEdgeData(env), env);

	DataSet<Vertex<Long, NullValue>> data = graph.getVertices();
	List<Vertex<Long, NullValue>> result = data.collect();

	expectedResult = "1,(null)\n" +
		"2,(null)\n" +
		"3,(null)\n" +
		"4,(null)\n" +
		"5,(null)\n";

	compareResultAsTuples(result, expectedResult);
}
 

@Test
public void testReverse() throws Exception {
	/*
	 * Test reverse()
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	DataSet<Edge<Long, Long>> data = graph.reverse().getEdges();
	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "2,1,12\n" +
		"3,1,13\n" +
		"3,2,23\n" +
		"4,3,34\n" +
		"5,3,35\n" +
		"5,4,45\n" +
		"1,5,51\n";

	compareResultAsTuples(result, expectedResult);
}
 

@Test
public void testOnTargetWithCustom() throws Exception {
	/*
     * Test joinWithEdgesOnTarget with a DataSet containing custom parametrised type input values
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	Graph<Long, Long, Long> res = graph.joinWithEdgesOnTarget(TestGraphUtils.getLongCustomTuple2TargetData(env),
		new CustomValueMapper());

	DataSet<Edge<Long, Long>> data = res.getEdges();
	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "1,2,10\n" +
		"1,3,20\n" +
		"2,3,20\n" +
		"3,4,40\n" +
		"3,5,35\n" +
		"4,5,45\n" +
		"5,1,51\n";

	compareResultAsTuples(result, expectedResult);
}
 

@Test
public void testWithStringValue() throws Exception {
	/*
	 * Test mapEdges() and change the value type to String
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	DataSet<Edge<Long, String>> mappedEdges = graph.mapEdges(new ToStringMapper()).getEdges();
	List<Edge<Long, String>> result = mappedEdges.collect();

	expectedResult = "1,2,string(12)\n" +
		"1,3,string(13)\n" +
		"2,3,string(23)\n" +
		"3,4,string(34)\n" +
		"3,5,string(35)\n" +
		"4,5,string(45)\n" +
		"5,1,string(51)\n";

	compareResultAsTuples(result, expectedResult);
}
 

@Test
public void testWithDoubleValueMapper() throws Exception {
	/*
	 * Test create() with edge dataset and a mapper that assigns a double constant as value
     */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	Graph<Long, Double, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongEdgeData(env),
		new AssignDoubleValueMapper(), env);

	DataSet<Vertex<Long, Double>> data = graph.getVertices();
	List<Vertex<Long, Double>> result = data.collect();

	expectedResult = "1,0.1\n" +
		"2,0.1\n" +
		"3,0.1\n" +
		"4,0.1\n" +
		"5,0.1\n";

	compareResultAsTuples(result, expectedResult);
}
 

/**
 * Test groupReduceOnNeighbors() -NeighborsFunction-
 * with an edge having a trgId that does not exist in the vertex DataSet.
 */
@Test
public void testGroupReduceOnNeighborsInvalidEdgeTrgId() throws Exception {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(PARALLELISM);

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
			TestGraphUtils.getLongLongEdgeInvalidSrcData(env), env);

	try {
		DataSet<Tuple2<Long, Long>> verticesWithSumOfAllNeighborValues =
				graph.reduceOnNeighbors(new SumNeighbors(), EdgeDirection.ALL);

		verticesWithSumOfAllNeighborValues.output(new DiscardingOutputFormat<>());
		env.execute();
	} catch (Exception e) {
		// We expect the job to fail with an exception
	}
}
 

@Test
public void testRemoveVertex() throws Exception {
	/*
	 * Test removeVertex() -- simple case
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
			TestGraphUtils.getLongLongEdgeData(env), env);
	graph = graph.removeVertex(new Vertex<>(5L, 5L));

	DataSet<Edge<Long, Long>> data = graph.getEdges();
	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "1,2,12\n" +
			"1,3,13\n" +
			"2,3,23\n" +
			"3,4,34\n";

	compareResultAsTuples(result, expectedResult);
}
 

@Test
public void testSumOfAllNeighborsNoValue() throws Exception {
	/*
	 * Get the sum of all neighbor values
	 * for each vertex
        */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	DataSet<Tuple2<Long, Long>> verticesWithSumOfAllNeighborValues =
		graph.reduceOnNeighbors(new SumNeighbors(), EdgeDirection.ALL);
	List<Tuple2<Long, Long>> result = verticesWithSumOfAllNeighborValues.collect();

	expectedResult = "1,10\n" +
		"2,4\n" +
		"3,12\n" +
		"4,8\n" +
		"5,8\n";

	compareResultAsTuples(result, expectedResult);
}
 

/**
 * Test groupReduceOnNeighbors() -NeighborsFunction-
 * with an edge having a trgId that does not exist in the vertex DataSet.
 */
@Test
public void testGroupReduceOnNeighborsInvalidEdgeTrgId() throws Exception {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(PARALLELISM);
	env.getConfig().disableSysoutLogging();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
			TestGraphUtils.getLongLongEdgeInvalidSrcData(env), env);

	try {
		DataSet<Tuple2<Long, Long>> verticesWithSumOfAllNeighborValues =
				graph.reduceOnNeighbors(new SumNeighbors(), EdgeDirection.ALL);

		verticesWithSumOfAllNeighborValues.output(new DiscardingOutputFormat<>());
		env.execute();
	} catch (Exception e) {
		// We expect the job to fail with an exception
	}
}
 

@Test
public void testSingleIteration() throws Exception {
	/*
	 * Test one iteration of label propagation example with a simple graph
	 */
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, NullValue> inputGraph = Graph.fromDataSet(
		LabelPropagationData.getDefaultVertexSet(env),
		LabelPropagationData.getDefaultEdgeDataSet(env), env);

	List<Vertex<Long, Long>> result = inputGraph
		.run(new LabelPropagation<>(1))
		.collect();

	expectedResult = LabelPropagationData.LABELS_AFTER_1_ITERATION;
	compareResultAsTuples(result, expectedResult);
}
 

@Test
public void testSumOfAllNeighborsNoValueMultipliedByTwoIdGreaterThanTwo() throws Exception {
	/*
	 * Get the sum of all neighbor values
	 * for each vertex with id greater than two as well as the same sum multiplied by two.
        */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	DataSet<Tuple2<Long, Long>> verticesWithSumOfAllNeighborValues =
		graph.groupReduceOnNeighbors(new SumAllNeighborsNoValueMultipliedByTwoIdGreaterThanTwo(), EdgeDirection.ALL);
	List<Tuple2<Long, Long>> result = verticesWithSumOfAllNeighborValues.collect();

	expectedResult = "3,12\n" +
		"3,24\n" +
		"4,8\n" +
		"4,16\n" +
		"5,8\n" +
		"5,16";

	compareResultAsTuples(result, expectedResult);
}
 

/**
 * Test groupReduceOnNeighbors() -NeighborsFunction-
 * with an edge having a srcId that does not exist in the vertex DataSet.
 */
@Test
public void testGroupReduceOnNeighborsInvalidEdgeSrcId() throws Exception {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(PARALLELISM);
	env.getConfig().disableSysoutLogging();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
			TestGraphUtils.getLongLongEdgeInvalidTrgData(env), env);

	try {
		DataSet<Tuple2<Long, Long>> verticesWithSumOfAllNeighborValues =
				graph.reduceOnNeighbors(new SumNeighbors(), EdgeDirection.ALL);

		verticesWithSumOfAllNeighborValues.output(new DiscardingOutputFormat<>());
		env.execute();
	} catch (Exception e) {
		// We expect the job to fail with an exception
	}
}
 

@SuppressWarnings("serial")
@Test
public void testSubGraph() throws Exception {
	/*
	 * Test subgraph:
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	DataSet<Edge<Long, Long>> data = graph.subgraph(
		new FilterFunction<Vertex<Long, Long>>() {
			public boolean filter(Vertex<Long, Long> vertex) throws Exception {
				return (vertex.getValue() > 2);
			}
		},
		new FilterFunction<Edge<Long, Long>>() {
			public boolean filter(Edge<Long, Long> edge) throws Exception {
				return (edge.getValue() > 34);
			}
		}).getEdges();

	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "3,5,35\n" +
		"4,5,45\n";

	compareResultAsTuples(result, expectedResult);
}
 

@Test
public void testGSAPageRankWithThreeIterationsAndNumOfVertices() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Double, Double> inputGraph = Graph.fromDataSet(
		PageRankData.getDefaultEdgeDataSet(env), new InitMapper(), env);

	List<Vertex<Long, Double>> result = inputGraph.run(new GSAPageRank<>(0.85, 3))
		.collect();

	compareWithDelta(result, 0.01);
}
 

@Test
public void testRemoveBothInvalidVertices() throws Exception {
	/*
	 * Test removeVertices() -- remove two invalid vertices
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
			TestGraphUtils.getLongLongEdgeData(env), env);
	List<Vertex<Long, Long>> verticesToBeRemoved = new ArrayList<>();
	verticesToBeRemoved.add(new Vertex<>(6L, 6L));
	verticesToBeRemoved.add(new Vertex<>(7L, 7L));

	graph = graph.removeVertices(verticesToBeRemoved);

	DataSet<Edge<Long, Long>> data = graph.getEdges();
	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "1,2,12\n" +
			"1,3,13\n" +
			"2,3,23\n" +
			"3,4,34\n" +
			"3,5,35\n" +
			"4,5,45\n" +
			"5,1,51\n";

	compareResultAsTuples(result, expectedResult);
}
 

@Test
public void testWithEdgesOnSource() throws Exception {
	/*
	 * Test joinWithEdgesOnSource with the input DataSet parameter identical
	 * to the edge DataSet
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	Graph<Long, Long, Long> res = graph.joinWithEdgesOnSource(graph.getEdges()
		.map(new ProjectSourceAndValueMapper()), new AddValuesMapper());

	DataSet<Edge<Long, Long>> data = res.getEdges();
	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "1,2,24\n" +
		"1,3,25\n" +
		"2,3,46\n" +
		"3,4,68\n" +
		"3,5,69\n" +
		"4,5,90\n" +
		"5,1,102\n";

	compareResultAsTuples(result, expectedResult);
}
 

@Test
public void testAddEdgesInvalidVertices() throws Exception {
	/*
	 * Test addEdges() -- the source and target vertices do not exist in the graph
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
			TestGraphUtils.getLongLongEdgeData(env), env);

	List<Edge<Long, Long>> edgesToBeAdded = new ArrayList<>();
	edgesToBeAdded.add(new Edge<>(6L, 1L, 61L));
	edgesToBeAdded.add(new Edge<>(7L, 1L, 71L));

	graph = graph.addEdges(edgesToBeAdded);

	DataSet<Edge<Long, Long>> data = graph.getEdges();
	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "1,2,12\n" +
			"1,3,13\n" +
			"2,3,23\n" +
			"3,4,34\n" +
			"3,5,35\n" +
			"4,5,45\n" +
			"5,1,51\n";

	compareResultAsTuples(result, expectedResult);
}
 

@SuppressWarnings("serial")
@Test
public void testPregelWithCombiner() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);
	// compose test program
	{

		DataSet<Vertex<Long, Long>> initialVertices = env.fromElements(
			new Tuple2<>(1L, 1L), new Tuple2<>(2L, 2L))
			.map(new Tuple2ToVertexMap<>());

		DataSet<Edge<Long, NullValue>> edges = env.fromElements(new Tuple2<>(1L, 2L))
			.map(new MapFunction<Tuple2<Long, Long>, Edge<Long, NullValue>>() {

				public Edge<Long, NullValue> map(Tuple2<Long, Long> edge) {
					return new Edge<>(edge.f0, edge.f1, NullValue.getInstance());
				}
			});

		Graph<Long, Long, NullValue> graph = Graph.fromDataSet(initialVertices, edges, env);

		DataSet<Vertex<Long, Long>> result = graph.runVertexCentricIteration(
			new CCCompute(), new CCCombiner(), 100).getVertices();

		result.output(new DiscardingOutputFormat<>());
	}

	Plan p = env.createProgramPlan("Pregel Connected Components");
	OptimizedPlan op = compileNoStats(p);

	// check the sink
	SinkPlanNode sink = op.getDataSinks().iterator().next();
	assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
	assertEquals(DEFAULT_PARALLELISM, sink.getParallelism());

	// check the iteration
	WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
	assertEquals(DEFAULT_PARALLELISM, iteration.getParallelism());

	// check the combiner
	SingleInputPlanNode combiner = (SingleInputPlanNode) iteration.getInput2().getSource();
	assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());

	// check the solution set delta
	PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
	assertTrue(ssDelta instanceof SingleInputPlanNode);

	SingleInputPlanNode ssFlatMap = (SingleInputPlanNode) ((SingleInputPlanNode) (ssDelta)).getInput().getSource();
	assertEquals(DEFAULT_PARALLELISM, ssFlatMap.getParallelism());
	assertEquals(ShipStrategyType.FORWARD, ssFlatMap.getInput().getShipStrategy());

	// check the computation coGroup
	DualInputPlanNode computationCoGroup = (DualInputPlanNode) (ssFlatMap.getInput().getSource());
	assertEquals(DEFAULT_PARALLELISM, computationCoGroup.getParallelism());
	assertEquals(ShipStrategyType.FORWARD, computationCoGroup.getInput1().getShipStrategy());
	assertEquals(ShipStrategyType.PARTITION_HASH, computationCoGroup.getInput2().getShipStrategy());
	assertTrue(computationCoGroup.getInput2().getTempMode().isCached());

	assertEquals(new FieldList(0), computationCoGroup.getInput2().getShipStrategyKeys());

	// check that the initial partitioning is pushed out of the loop
	assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
	assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
}
 

/**
 * Convert a bipartite graph into an undirected graph that contains only top vertices. An edge between two vertices
 * in the new graph will exist only if the original bipartite graph contains a bottom vertex they are both
 * connected to.
 *
 * <p>The simple projection performs a single join and returns edges containing the bipartite edge values.
 *
 * <p>Note: KT must override .equals(). This requirement may be removed in a future release.
 *
 * @return simple top projection of the bipartite graph
 */
public Graph<KT, VVT, Tuple2<EV, EV>> projectionTopSimple() {
	DataSet<Edge<KT, Tuple2<EV, EV>>> newEdges = edges.join(edges)
		.where(1)
		.equalTo(1)
		.with(new ProjectionTopSimple<>())
			.name("Simple top projection");

	return Graph.fromDataSet(topVertices, newEdges, context);
}