Java Code Examples for org.apache.flink.graph.Graph#run()

@Override
public DataSet<Edge<K, Tuple2<EV, LongValue>>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// s, d(s)
	DataSet<Vertex<K, LongValue>> vertexDegrees = input
		.run(new VertexDegree<K, VV, EV>()
			.setReduceOnTargetId(reduceOnTargetId.get())
			.setParallelism(parallelism));

	// s, t, d(s)
	return input.getEdges()
		.join(vertexDegrees, JoinHint.REPARTITION_HASH_SECOND)
		.where(0)
		.equalTo(0)
		.with(new JoinEdgeWithVertexDegree<>())
			.setParallelism(parallelism)
			.name("Edge source degree");
}
 

@Override
protected void testProgram() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Long> vertexIds = env.generateSequence(1, NUM_VERTICES);
	DataSet<String> edgeString = env.fromElements(ConnectedComponentsData.getRandomOddEvenEdges(NUM_EDGES, NUM_VERTICES, SEED).split("\n"));

	DataSet<Edge<Long, NullValue>> edges = edgeString.map(new EdgeParser());

	DataSet<Vertex<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());

	Graph<Long, Long, NullValue> graph = Graph.fromDataSet(initialVertices, edges, env);

	DataSet<Vertex<Long, Long>> result = graph.run(new ConnectedComponents<>(100));

	result.writeAsCsv(resultPath, "\n", " ");
	env.execute();
}
 

@Override
public DataSet<Edge<K, Tuple2<EV, Degrees>>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// s, d(s)
	DataSet<Vertex<K, Degrees>> vertexDegrees = input
		.run(new VertexDegrees<K, VV, EV>()
			.setParallelism(parallelism));

	// s, t, d(s)
	return input.getEdges()
		.join(vertexDegrees, JoinHint.REPARTITION_HASH_SECOND)
		.where(0)
		.equalTo(0)
		.with(new JoinEdgeWithVertexDegree<>())
			.setParallelism(parallelism)
			.name("Edge source degrees");
}
 

@Override
public TriadicCensus<K, VV, EV> run(Graph<K, VV, EV> input)
		throws Exception {
	super.run(input);

	triangleCount = new Count<>();

	DataSet<TriangleListing.Result<K>> triangles = input
		.run(new TriangleListing<K, VV, EV>()
			.setSortTriangleVertices(false)
			.setParallelism(parallelism));

	triangleCount.run(triangles);

	vertexMetrics = new VertexMetrics<K, VV, EV>()
		.setParallelism(parallelism);

	input.run(vertexMetrics);

	return this;
}
 

@Override
public DataSet<Edge<K, Tuple2<EV, LongValue>>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// t, d(t)
	DataSet<Vertex<K, LongValue>> vertexDegrees = input
		.run(new VertexDegree<K, VV, EV>()
			.setReduceOnTargetId(!reduceOnSourceId.get())
			.setParallelism(parallelism));

	// s, t, d(t)
	return input.getEdges()
		.join(vertexDegrees, JoinHint.REPARTITION_HASH_SECOND)
		.where(1)
		.equalTo(0)
		.with(new JoinEdgeWithVertexDegree<>())
			.setParallelism(parallelism)
			.name("Edge target degree");
}
 

@Override
public DataSet<Edge<K, Tuple3<EV, LongValue, LongValue>>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// s, t, d(s)
	DataSet<Edge<K, Tuple2<EV, LongValue>>> edgeSourceDegrees = input
		.run(new EdgeSourceDegree<K, VV, EV>()
			.setReduceOnTargetId(reduceOnTargetId.get())
			.setParallelism(parallelism));

	// t, d(t)
	DataSet<Vertex<K, LongValue>> vertexDegrees = input
		.run(new VertexDegree<K, VV, EV>()
			.setReduceOnTargetId(reduceOnTargetId.get())
			.setParallelism(parallelism));

	// s, t, (d(s), d(t))
	return edgeSourceDegrees
		.join(vertexDegrees, JoinHint.REPARTITION_HASH_SECOND)
		.where(1)
		.equalTo(0)
		.with(new JoinEdgeDegreeWithVertexDegree<>())
			.setParallelism(parallelism)
			.name("Edge target degree");
}
 

@Override
public GlobalClusteringCoefficient<K, VV, EV> run(Graph<K, VV, EV> input)
		throws Exception {
	super.run(input);

	triangleCount = new Count<>();

	DataSet<TriangleListing.Result<K>> triangles = input
		.run(new TriangleListing<K, VV, EV>()
			.setSortTriangleVertices(false)
			.setParallelism(parallelism));

	triangleCount.run(triangles);

	vertexMetrics = new VertexMetrics<K, VV, EV>()
		.setParallelism(parallelism);

	input.run(vertexMetrics);

	return this;
}
 

@Override
public AverageClusteringCoefficient<K, VV, EV> run(Graph<K, VV, EV> input)
		throws Exception {
	super.run(input);

	DataSet<LocalClusteringCoefficient.Result<K>> localClusteringCoefficient = input
		.run(new LocalClusteringCoefficient<K, VV, EV>()
			.setParallelism(parallelism));

	averageClusteringCoefficientHelper = new AverageClusteringCoefficientHelper<>();

	localClusteringCoefficient
		.output(averageClusteringCoefficientHelper)
			.name("Average clustering coefficient");

	return this;
}
 

@Override
public DataSet<Edge<K, Tuple3<EV, Degrees, Degrees>>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// s, t, d(s)
	DataSet<Edge<K, Tuple2<EV, Degrees>>> edgeSourceDegrees = input
		.run(new EdgeSourceDegrees<K, VV, EV>()
			.setParallelism(parallelism));

	// t, d(t)
	DataSet<Vertex<K, Degrees>> vertexDegrees = input
		.run(new VertexDegrees<K, VV, EV>()
			.setParallelism(parallelism));

	// s, t, (d(s), d(t))
	return edgeSourceDegrees
		.join(vertexDegrees, JoinHint.REPARTITION_HASH_SECOND)
		.where(1)
		.equalTo(0)
		.with(new JoinEdgeDegreeWithVertexDegree<>())
			.setParallelism(parallelism)
			.name("Edge target degree");
}
 

@Override
public DataSet<Edge<K, Tuple2<EV, LongValue>>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// t, d(t)
	DataSet<Vertex<K, LongValue>> vertexDegrees = input
		.run(new VertexDegree<K, VV, EV>()
			.setReduceOnTargetId(!reduceOnSourceId.get())
			.setParallelism(parallelism));

	// s, t, d(t)
	return input.getEdges()
		.join(vertexDegrees, JoinHint.REPARTITION_HASH_SECOND)
		.where(1)
		.equalTo(0)
		.with(new JoinEdgeWithVertexDegree<>())
			.setParallelism(parallelism)
			.name("Edge target degree");
}
 

@Override
public EdgeMetrics<K, VV, EV> run(Graph<K, VV, EV> input)
		throws Exception {
	super.run(input);

	// s, t, (d(s), d(t))
	DataSet<Edge<K, Tuple3<EV, LongValue, LongValue>>> edgeDegreePair = input
		.run(new EdgeDegreePair<K, VV, EV>()
			.setReduceOnTargetId(reduceOnTargetId)
			.setParallelism(parallelism));

	// s, d(s), count of (u, v) where deg(u) < deg(v) or (deg(u) == deg(v) and u < v)
	DataSet<Tuple3<K, LongValue, LongValue>> edgeStats = edgeDegreePair
		.map(new EdgeStats<>())
			.setParallelism(parallelism)
			.name("Edge stats")
		.groupBy(0)
		.reduce(new SumEdgeStats<>())
		.setCombineHint(CombineHint.HASH)
			.setParallelism(parallelism)
			.name("Sum edge stats");

	edgeMetricsHelper = new EdgeMetricsHelper<>();

	edgeStats
		.output(edgeMetricsHelper)
			.setParallelism(parallelism)
			.name("Edge metrics");

	return this;
}
 

@Override
public DataSet plan(Graph<K, VV, EV> graph) throws Exception {
	return graph
		.run(new org.apache.flink.graph.library.linkanalysis.PageRank<K, VV, EV>(
				dampingFactor.getValue(),
				iterationConvergence.getValue().iterations,
				iterationConvergence.getValue().convergenceThreshold)
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices.getValue())
			.setParallelism(parallelism.getValue().intValue()));
}
 

@Override
public EdgeMetrics<K, VV, EV> run(Graph<K, VV, EV> input)
		throws Exception {
	super.run(input);

	// s, t, (d(s), d(t))
	DataSet<Edge<K, Tuple3<EV, LongValue, LongValue>>> edgeDegreePair = input
		.run(new EdgeDegreePair<K, VV, EV>()
			.setReduceOnTargetId(reduceOnTargetId)
			.setParallelism(parallelism));

	// s, d(s), count of (u, v) where deg(u) < deg(v) or (deg(u) == deg(v) and u < v)
	DataSet<Tuple3<K, LongValue, LongValue>> edgeStats = edgeDegreePair
		.map(new EdgeStats<>())
			.setParallelism(parallelism)
			.name("Edge stats")
		.groupBy(0)
		.reduce(new SumEdgeStats<>())
		.setCombineHint(CombineHint.HASH)
			.setParallelism(parallelism)
			.name("Sum edge stats");

	edgeMetricsHelper = new EdgeMetricsHelper<>();

	edgeStats
		.output(edgeMetricsHelper)
			.setParallelism(parallelism)
			.name("Edge metrics");

	return this;
}
 

@Override
public DataSet plan(Graph<K, VV, EV> graph) throws Exception {
	int parallelism = this.parallelism.getValue().intValue();

	switch (order.getValue()) {
		case DIRECTED:
			if (computeTriadicCensus.getValue()) {
				triadicCensus = graph
					.run(new org.apache.flink.graph.library.clustering.directed.TriadicCensus<K, VV, EV>()
						.setParallelism(parallelism));
			}

			@SuppressWarnings("unchecked")
			DataSet<PrintableResult> directedResult = (DataSet<PrintableResult>) (DataSet<?>) graph
				.run(new org.apache.flink.graph.library.clustering.directed.TriangleListing<K, VV, EV>()
					.setPermuteResults(permuteResults.getValue())
					.setSortTriangleVertices(sortTriangleVertices.getValue())
					.setParallelism(parallelism));
			return directedResult;

		case UNDIRECTED:
			if (computeTriadicCensus.getValue()) {
				triadicCensus = graph
					.run(new org.apache.flink.graph.library.clustering.undirected.TriadicCensus<K, VV, EV>()
						.setParallelism(parallelism));
			}

			@SuppressWarnings("unchecked")
			DataSet<PrintableResult> undirectedResult = (DataSet<PrintableResult>) (DataSet<?>) graph
				.run(new org.apache.flink.graph.library.clustering.undirected.TriangleListing<K, VV, EV>()
					.setPermuteResults(permuteResults.getValue())
					.setSortTriangleVertices(sortTriangleVertices.getValue())
					.setParallelism(parallelism));
			return undirectedResult;

		default:
			throw new RuntimeException("Unknown order: " + order);
	}
}
 

@Override
public DataSet plan(Graph<K, VV, EV> graph) throws Exception {
	return graph
		.run(new org.apache.flink.graph.library.similarity.AdamicAdar<K, VV, EV>()
			.setMinimumRatio(minRatio.getValue().floatValue())
			.setMinimumScore(minScore.getValue().floatValue())
			.setMirrorResults(mirrorResults.getValue())
			.setParallelism(parallelism.getValue().intValue()));
}
 

@Override
public DataSet plan(Graph<K, VV, EV> graph) throws Exception {
	return graph
		.run(new org.apache.flink.graph.library.linkanalysis.PageRank<K, VV, EV>(
				dampingFactor.getValue(),
				iterationConvergence.getValue().iterations,
				iterationConvergence.getValue().convergenceThreshold)
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices.getValue())
			.setParallelism(parallelism.getValue().intValue()));
}
 

/**
 * Validate a test where each result has the same values.
 *
 * @param graph input graph
 * @param count number of results
 * @param score result score
 * @param <T> graph ID type
 * @throws Exception on error
 */
private static <T extends CopyableValue<T>> void validate(
		Graph<T, NullValue, NullValue> graph, long count, double score) throws Exception {
	DataSet<Result<T>> aa = graph
		.run(new AdamicAdar<>());

	List<Result<T>> results = aa.collect();

	assertEquals(count, results.size());

	for (Result<T> result : results) {
		assertEquals(score, result.getAdamicAdarScore().getValue(), ACCURACY);
	}
}
 

@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// u, v, w
	DataSet<TriangleListing.Result<K>> triangles = input
		.run(new TriangleListing<K, VV, EV>()
			.setParallelism(parallelism));

	// u, 1
	DataSet<Tuple2<K, LongValue>> triangleVertices = triangles
		.flatMap(new SplitTriangles<>())
			.name("Split triangle vertices");

	// u, triangle count
	DataSet<Tuple2<K, LongValue>> vertexTriangleCount = triangleVertices
		.groupBy(0)
		.reduce(new CountTriangles<>())
		.setCombineHint(CombineHint.HASH)
			.name("Count triangles")
			.setParallelism(parallelism);

	// u, deg(u)
	DataSet<Vertex<K, LongValue>> vertexDegree = input
		.run(new VertexDegree<K, VV, EV>()
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices.get())
			.setParallelism(parallelism));

	// u, deg(u), triangle count
	return vertexDegree
		.leftOuterJoin(vertexTriangleCount)
		.where(0)
		.equalTo(0)
		.with(new JoinVertexDegreeWithTriangleCount<>())
			.setParallelism(parallelism)
			.name("Clustering coefficient");
}
 

@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// u, v, bitmask where u < v
	DataSet<Tuple3<K, K, ByteValue>> filteredByID = input
		.getEdges()
		.map(new OrderByID<>())
			.setParallelism(parallelism)
			.name("Order by ID")
		.groupBy(0, 1)
		.reduceGroup(new ReduceBitmask<>())
			.setParallelism(parallelism)
			.name("Flatten by ID");

	// u, v, (deg(u), deg(v))
	DataSet<Edge<K, Tuple3<EV, Degrees, Degrees>>> pairDegrees = input
		.run(new EdgeDegreesPair<K, VV, EV>()
			.setParallelism(parallelism));

	// u, v, bitmask where deg(u) < deg(v) or (deg(u) == deg(v) and u < v)
	DataSet<Tuple3<K, K, ByteValue>> filteredByDegree = pairDegrees
		.map(new OrderByDegree<>())
			.setParallelism(parallelism)
			.name("Order by degree")
		.groupBy(0, 1)
		.reduceGroup(new ReduceBitmask<>())
			.setParallelism(parallelism)
			.name("Flatten by degree");

	// u, v, w, bitmask where (u, v) and (u, w) are edges in graph
	DataSet<Tuple4<K, K, K, ByteValue>> triplets = filteredByDegree
		.groupBy(0)
		.sortGroup(1, Order.ASCENDING)
		.reduceGroup(new GenerateTriplets<>())
			.name("Generate triplets");

	// u, v, w, bitmask where (u, v), (u, w), and (v, w) are edges in graph
	DataSet<Result<K>> triangles = triplets
		.join(filteredByID, JoinOperatorBase.JoinHint.REPARTITION_HASH_SECOND)
		.where(1, 2)
		.equalTo(0, 1)
		.with(new ProjectTriangles<>())
			.name("Triangle listing");

	if (permuteResults) {
		triangles = triangles
			.flatMap(new PermuteResult<>())
				.name("Permute triangle vertices");
	} else if (sortTriangleVertices.get()) {
		triangles = triangles
			.map(new SortTriangleVertices<>())
				.name("Sort triangle vertices");
	}

	return triangles;
}
 

@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// vertex degree
	DataSet<Vertex<K, Degrees>> vertexDegree = input
		.run(new VertexDegrees<K, VV, EV>()
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices)
			.setParallelism(parallelism));

	// vertex count
	DataSet<LongValue> vertexCount = GraphUtils.count(vertexDegree);

	// s, t, d(s)
	DataSet<Edge<K, LongValue>> edgeSourceDegree = input
		.run(new EdgeSourceDegrees<K, VV, EV>()
			.setParallelism(parallelism))
		.map(new ExtractSourceDegree<>())
			.setParallelism(parallelism)
			.name("Extract source degree");

	// vertices with zero in-edges
	DataSet<Tuple2<K, DoubleValue>> sourceVertices = vertexDegree
		.flatMap(new InitializeSourceVertices<>())
			.setParallelism(parallelism)
			.name("Initialize source vertex scores");

	// s, initial pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> initialScores = vertexDegree
		.map(new InitializeVertexScores<>())
		.withBroadcastSet(vertexCount, VERTEX_COUNT)
			.setParallelism(parallelism)
			.name("Initialize scores");

	IterativeDataSet<Tuple2<K, DoubleValue>> iterative = initialScores
		.iterate(maxIterations)
		.setParallelism(parallelism);

	// s, projected pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> vertexScores = iterative
		.coGroup(edgeSourceDegree)
		.where(0)
		.equalTo(0)
		.with(new SendScore<>())
			.setParallelism(parallelism)
			.name("Send score")
		.groupBy(0)
		.reduce(new SumScore<>())
		.setCombineHint(CombineHint.HASH)
			.setParallelism(parallelism)
			.name("Sum");

	// ignored ID, total pagerank
	DataSet<Tuple2<K, DoubleValue>> sumOfScores = vertexScores
		.reduce(new SumVertexScores<>())
			.setParallelism(parallelism)
			.name("Sum");

	// s, adjusted pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> adjustedScores = vertexScores
		.union(sourceVertices)
			.name("Union with source vertices")
		.map(new AdjustScores<>(dampingFactor))
			.withBroadcastSet(sumOfScores, SUM_OF_SCORES)
			.withBroadcastSet(vertexCount, VERTEX_COUNT)
				.setParallelism(parallelism)
				.name("Adjust scores");

	DataSet<Tuple2<K, DoubleValue>> passThrough;

	if (convergenceThreshold < Double.MAX_VALUE) {
		passThrough = iterative
			.join(adjustedScores)
			.where(0)
			.equalTo(0)
			.with(new ChangeInScores<>())
				.setParallelism(parallelism)
				.name("Change in scores");

		iterative.registerAggregationConvergenceCriterion(CHANGE_IN_SCORES, new DoubleSumAggregator(), new ScoreConvergence(convergenceThreshold));
	} else {
		passThrough = adjustedScores;
	}

	return iterative
		.closeWith(passThrough)
		.map(new TranslateResult<>())
			.setParallelism(parallelism)
			.name("Map result");
}