Java Code Examples for org.apache.flink.graph.utils.GraphUtils#count()

The following examples show how to use org.apache.flink.graph.utils.GraphUtils#count() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ScatterGatherIteration.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Creates the operator that represents this scatter-gather graph computation.
 *
 * @return The operator that represents this scatter-gather graph computation.
 */
@Override
public DataSet<Vertex<K, VV>> createResult() {
	if (this.initialVertices == null) {
		throw new IllegalStateException("The input data set has not been set.");
	}

	// prepare some type information
	TypeInformation<K> keyType = ((TupleTypeInfo<?>) initialVertices.getType()).getTypeAt(0);
	TypeInformation<Tuple2<K, Message>> messageTypeInfo = new TupleTypeInfo<>(keyType, messageType);

	// create a graph
	Graph<K, VV, EV> graph =
			Graph.fromDataSet(initialVertices, edgesWithValue, initialVertices.getExecutionEnvironment());

	// check whether the numVertices option is set and, if so, compute the total number of vertices
	// and set it within the scatter and gather functions

	DataSet<LongValue> numberOfVertices = null;
	if (this.configuration != null && this.configuration.isOptNumVertices()) {
		try {
			numberOfVertices = GraphUtils.count(this.initialVertices);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	if (this.configuration != null) {
		scatterFunction.setDirection(this.configuration.getDirection());
	} else {
		scatterFunction.setDirection(EdgeDirection.OUT);
	}

	// retrieve the direction in which the updates are made and in which the messages are sent
	EdgeDirection messagingDirection = scatterFunction.getDirection();

	// check whether the degrees option is set and, if so, compute the in and the out degrees and
	// add them to the vertex value
	if (this.configuration != null && this.configuration.isOptDegrees()) {
		return createResultVerticesWithDegrees(graph, messagingDirection, messageTypeInfo, numberOfVertices);
	} else {
		return createResultSimpleVertex(messagingDirection, messageTypeInfo, numberOfVertices);
	}
}

Example 2

Source File: PageRank.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// vertex degree
	DataSet<Vertex<K, Degrees>> vertexDegree = input
		.run(new VertexDegrees<K, VV, EV>()
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices)
			.setParallelism(parallelism));

	// vertex count
	DataSet<LongValue> vertexCount = GraphUtils.count(vertexDegree);

	// s, t, d(s)
	DataSet<Edge<K, LongValue>> edgeSourceDegree = input
		.run(new EdgeSourceDegrees<K, VV, EV>()
			.setParallelism(parallelism))
		.map(new ExtractSourceDegree<>())
			.setParallelism(parallelism)
			.name("Extract source degree");

	// vertices with zero in-edges
	DataSet<Tuple2<K, DoubleValue>> sourceVertices = vertexDegree
		.flatMap(new InitializeSourceVertices<>())
			.setParallelism(parallelism)
			.name("Initialize source vertex scores");

	// s, initial pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> initialScores = vertexDegree
		.map(new InitializeVertexScores<>())
		.withBroadcastSet(vertexCount, VERTEX_COUNT)
			.setParallelism(parallelism)
			.name("Initialize scores");

	IterativeDataSet<Tuple2<K, DoubleValue>> iterative = initialScores
		.iterate(maxIterations)
		.setParallelism(parallelism);

	// s, projected pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> vertexScores = iterative
		.coGroup(edgeSourceDegree)
		.where(0)
		.equalTo(0)
		.with(new SendScore<>())
			.setParallelism(parallelism)
			.name("Send score")
		.groupBy(0)
		.reduce(new SumScore<>())
		.setCombineHint(CombineHint.HASH)
			.setParallelism(parallelism)
			.name("Sum");

	// ignored ID, total pagerank
	DataSet<Tuple2<K, DoubleValue>> sumOfScores = vertexScores
		.reduce(new SumVertexScores<>())
			.setParallelism(parallelism)
			.name("Sum");

	// s, adjusted pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> adjustedScores = vertexScores
		.union(sourceVertices)
			.name("Union with source vertices")
		.map(new AdjustScores<>(dampingFactor))
			.withBroadcastSet(sumOfScores, SUM_OF_SCORES)
			.withBroadcastSet(vertexCount, VERTEX_COUNT)
				.setParallelism(parallelism)
				.name("Adjust scores");

	DataSet<Tuple2<K, DoubleValue>> passThrough;

	if (convergenceThreshold < Double.MAX_VALUE) {
		passThrough = iterative
			.join(adjustedScores)
			.where(0)
			.equalTo(0)
			.with(new ChangeInScores<>())
				.setParallelism(parallelism)
				.name("Change in scores");

		iterative.registerAggregationConvergenceCriterion(CHANGE_IN_SCORES, new DoubleSumAggregator(), new ScoreConvergence(convergenceThreshold));
	} else {
		passThrough = adjustedScores;
	}

	return iterative
		.closeWith(passThrough)
		.map(new TranslateResult<>())
			.setParallelism(parallelism)
			.name("Map result");
}

Example 3

Source File: ScatterGatherIteration.java From flink with Apache License 2.0

4 votes

/**
 * Creates the operator that represents this scatter-gather graph computation.
 *
 * @return The operator that represents this scatter-gather graph computation.
 */
@Override
public DataSet<Vertex<K, VV>> createResult() {
	if (this.initialVertices == null) {
		throw new IllegalStateException("The input data set has not been set.");
	}

	// prepare some type information
	TypeInformation<K> keyType = ((TupleTypeInfo<?>) initialVertices.getType()).getTypeAt(0);
	TypeInformation<Tuple2<K, Message>> messageTypeInfo = new TupleTypeInfo<>(keyType, messageType);

	// create a graph
	Graph<K, VV, EV> graph =
			Graph.fromDataSet(initialVertices, edgesWithValue, initialVertices.getExecutionEnvironment());

	// check whether the numVertices option is set and, if so, compute the total number of vertices
	// and set it within the scatter and gather functions

	DataSet<LongValue> numberOfVertices = null;
	if (this.configuration != null && this.configuration.isOptNumVertices()) {
		try {
			numberOfVertices = GraphUtils.count(this.initialVertices);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	if (this.configuration != null) {
		scatterFunction.setDirection(this.configuration.getDirection());
	} else {
		scatterFunction.setDirection(EdgeDirection.OUT);
	}

	// retrieve the direction in which the updates are made and in which the messages are sent
	EdgeDirection messagingDirection = scatterFunction.getDirection();

	// check whether the degrees option is set and, if so, compute the in and the out degrees and
	// add them to the vertex value
	if (this.configuration != null && this.configuration.isOptDegrees()) {
		return createResultVerticesWithDegrees(graph, messagingDirection, messageTypeInfo, numberOfVertices);
	} else {
		return createResultSimpleVertex(messagingDirection, messageTypeInfo, numberOfVertices);
	}
}

Example 4

Source File: PageRank.java From flink with Apache License 2.0

4 votes

@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// vertex degree
	DataSet<Vertex<K, Degrees>> vertexDegree = input
		.run(new VertexDegrees<K, VV, EV>()
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices)
			.setParallelism(parallelism));

	// vertex count
	DataSet<LongValue> vertexCount = GraphUtils.count(vertexDegree);

	// s, t, d(s)
	DataSet<Edge<K, LongValue>> edgeSourceDegree = input
		.run(new EdgeSourceDegrees<K, VV, EV>()
			.setParallelism(parallelism))
		.map(new ExtractSourceDegree<>())
			.setParallelism(parallelism)
			.name("Extract source degree");

	// vertices with zero in-edges
	DataSet<Tuple2<K, DoubleValue>> sourceVertices = vertexDegree
		.flatMap(new InitializeSourceVertices<>())
			.setParallelism(parallelism)
			.name("Initialize source vertex scores");

	// s, initial pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> initialScores = vertexDegree
		.map(new InitializeVertexScores<>())
		.withBroadcastSet(vertexCount, VERTEX_COUNT)
			.setParallelism(parallelism)
			.name("Initialize scores");

	IterativeDataSet<Tuple2<K, DoubleValue>> iterative = initialScores
		.iterate(maxIterations)
		.setParallelism(parallelism);

	// s, projected pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> vertexScores = iterative
		.coGroup(edgeSourceDegree)
		.where(0)
		.equalTo(0)
		.with(new SendScore<>())
			.setParallelism(parallelism)
			.name("Send score")
		.groupBy(0)
		.reduce(new SumScore<>())
		.setCombineHint(CombineHint.HASH)
			.setParallelism(parallelism)
			.name("Sum");

	// ignored ID, total pagerank
	DataSet<Tuple2<K, DoubleValue>> sumOfScores = vertexScores
		.reduce(new SumVertexScores<>())
			.setParallelism(parallelism)
			.name("Sum");

	// s, adjusted pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> adjustedScores = vertexScores
		.union(sourceVertices)
			.name("Union with source vertices")
		.map(new AdjustScores<>(dampingFactor))
			.withBroadcastSet(sumOfScores, SUM_OF_SCORES)
			.withBroadcastSet(vertexCount, VERTEX_COUNT)
				.setParallelism(parallelism)
				.name("Adjust scores");

	DataSet<Tuple2<K, DoubleValue>> passThrough;

	if (convergenceThreshold < Double.MAX_VALUE) {
		passThrough = iterative
			.join(adjustedScores)
			.where(0)
			.equalTo(0)
			.with(new ChangeInScores<>())
				.setParallelism(parallelism)
				.name("Change in scores");

		iterative.registerAggregationConvergenceCriterion(CHANGE_IN_SCORES, new DoubleSumAggregator(), new ScoreConvergence(convergenceThreshold));
	} else {
		passThrough = adjustedScores;
	}

	return iterative
		.closeWith(passThrough)
		.map(new TranslateResult<>())
			.setParallelism(parallelism)
			.name("Map result");
}

Example 5

Source File: ScatterGatherIteration.java From flink with Apache License 2.0

4 votes

/**
 * Creates the operator that represents this scatter-gather graph computation.
 *
 * @return The operator that represents this scatter-gather graph computation.
 */
@Override
public DataSet<Vertex<K, VV>> createResult() {
	if (this.initialVertices == null) {
		throw new IllegalStateException("The input data set has not been set.");
	}

	// prepare some type information
	TypeInformation<K> keyType = ((TupleTypeInfo<?>) initialVertices.getType()).getTypeAt(0);
	TypeInformation<Tuple2<K, Message>> messageTypeInfo = new TupleTypeInfo<>(keyType, messageType);

	// create a graph
	Graph<K, VV, EV> graph =
			Graph.fromDataSet(initialVertices, edgesWithValue, initialVertices.getExecutionEnvironment());

	// check whether the numVertices option is set and, if so, compute the total number of vertices
	// and set it within the scatter and gather functions

	DataSet<LongValue> numberOfVertices = null;
	if (this.configuration != null && this.configuration.isOptNumVertices()) {
		try {
			numberOfVertices = GraphUtils.count(this.initialVertices);
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	if (this.configuration != null) {
		scatterFunction.setDirection(this.configuration.getDirection());
	} else {
		scatterFunction.setDirection(EdgeDirection.OUT);
	}

	// retrieve the direction in which the updates are made and in which the messages are sent
	EdgeDirection messagingDirection = scatterFunction.getDirection();

	// check whether the degrees option is set and, if so, compute the in and the out degrees and
	// add them to the vertex value
	if (this.configuration != null && this.configuration.isOptDegrees()) {
		return createResultVerticesWithDegrees(graph, messagingDirection, messageTypeInfo, numberOfVertices);
	} else {
		return createResultSimpleVertex(messagingDirection, messageTypeInfo, numberOfVertices);
	}
}

Example 6

Source File: PageRank.java From flink with Apache License 2.0

4 votes

@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// vertex degree
	DataSet<Vertex<K, Degrees>> vertexDegree = input
		.run(new VertexDegrees<K, VV, EV>()
			.setIncludeZeroDegreeVertices(includeZeroDegreeVertices)
			.setParallelism(parallelism));

	// vertex count
	DataSet<LongValue> vertexCount = GraphUtils.count(vertexDegree);

	// s, t, d(s)
	DataSet<Edge<K, LongValue>> edgeSourceDegree = input
		.run(new EdgeSourceDegrees<K, VV, EV>()
			.setParallelism(parallelism))
		.map(new ExtractSourceDegree<>())
			.setParallelism(parallelism)
			.name("Extract source degree");

	// vertices with zero in-edges
	DataSet<Tuple2<K, DoubleValue>> sourceVertices = vertexDegree
		.flatMap(new InitializeSourceVertices<>())
			.setParallelism(parallelism)
			.name("Initialize source vertex scores");

	// s, initial pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> initialScores = vertexDegree
		.map(new InitializeVertexScores<>())
		.withBroadcastSet(vertexCount, VERTEX_COUNT)
			.setParallelism(parallelism)
			.name("Initialize scores");

	IterativeDataSet<Tuple2<K, DoubleValue>> iterative = initialScores
		.iterate(maxIterations)
		.setParallelism(parallelism);

	// s, projected pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> vertexScores = iterative
		.coGroup(edgeSourceDegree)
		.where(0)
		.equalTo(0)
		.with(new SendScore<>())
			.setParallelism(parallelism)
			.name("Send score")
		.groupBy(0)
		.reduce(new SumScore<>())
		.setCombineHint(CombineHint.HASH)
			.setParallelism(parallelism)
			.name("Sum");

	// ignored ID, total pagerank
	DataSet<Tuple2<K, DoubleValue>> sumOfScores = vertexScores
		.reduce(new SumVertexScores<>())
			.setParallelism(parallelism)
			.name("Sum");

	// s, adjusted pagerank(s)
	DataSet<Tuple2<K, DoubleValue>> adjustedScores = vertexScores
		.union(sourceVertices)
			.name("Union with source vertices")
		.map(new AdjustScores<>(dampingFactor))
			.withBroadcastSet(sumOfScores, SUM_OF_SCORES)
			.withBroadcastSet(vertexCount, VERTEX_COUNT)
				.setParallelism(parallelism)
				.name("Adjust scores");

	DataSet<Tuple2<K, DoubleValue>> passThrough;

	if (convergenceThreshold < Double.MAX_VALUE) {
		passThrough = iterative
			.join(adjustedScores)
			.where(0)
			.equalTo(0)
			.with(new ChangeInScores<>())
				.setParallelism(parallelism)
				.name("Change in scores");

		iterative.registerAggregationConvergenceCriterion(CHANGE_IN_SCORES, new DoubleSumAggregator(), new ScoreConvergence(convergenceThreshold));
	} else {
		passThrough = adjustedScores;
	}

	return iterative
		.closeWith(passThrough)
		.map(new TranslateResult<>())
			.setParallelism(parallelism)
			.name("Map result");
}