org.apache.flink.api.common.operators.base.JoinOperatorBase Java Examples

The following examples show how to use org.apache.flink.api.common.operators.base.JoinOperatorBase. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JoinCancelingITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private void executeTaskWithGenerator(
		JoinFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> joiner,
		int keys, int vals, int msecsTillCanceling, int maxTimeTillCanceled) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple2<Integer, Integer>> input1 = env.createInput(new UniformIntTupleGeneratorInputFormat(keys, vals));
	DataSet<Tuple2<Integer, Integer>> input2 = env.createInput(new UniformIntTupleGeneratorInputFormat(keys, vals));

	input1.join(input2, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE)
			.where(0)
			.equalTo(0)
			.with(joiner)
			.output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());

	env.setParallelism(PARALLELISM);

	runAndCancelJob(env.createProgramPlan(), msecsTillCanceling, maxTimeTillCanceled);
}
Example #2
Source File: JoinCancelingITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCancelSortMatchWhileDoingHeavySorting() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	HeavyCompareGeneratorInputFormat input = new HeavyCompareGeneratorInputFormat(100);
	DataSet<Tuple2<HeavyCompare, Integer>> input1 = env.createInput(input);
	DataSet<Tuple2<HeavyCompare, Integer>> input2 = env.createInput(input);

	input1.join(input2, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE)
			.where(0)
			.equalTo(0)
			.with(new JoinFunction<Tuple2<HeavyCompare, Integer>, Tuple2<HeavyCompare, Integer>, Tuple2<HeavyCompare, Integer>>() {
				@Override
				public Tuple2<HeavyCompare, Integer> join(
					Tuple2<HeavyCompare, Integer> first,
					Tuple2<HeavyCompare, Integer> second) throws Exception {
					throw new Exception("Job should be canceled in sort-merge phase, never run here ...");
				}
			})
			.output(new DiscardingOutputFormat<Tuple2<HeavyCompare, Integer>>());

	runAndCancelJob(env.createProgramPlan(), 30 * 1000, 60 * 1000);
}
Example #3
Source File: JoinCancelingITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCancelSortMatchWhileDoingHeavySorting() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	HeavyCompareGeneratorInputFormat input = new HeavyCompareGeneratorInputFormat(100);
	DataSet<Tuple2<HeavyCompare, Integer>> input1 = env.createInput(input);
	DataSet<Tuple2<HeavyCompare, Integer>> input2 = env.createInput(input);

	input1.join(input2, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE)
			.where(0)
			.equalTo(0)
			.with(new JoinFunction<Tuple2<HeavyCompare, Integer>, Tuple2<HeavyCompare, Integer>, Tuple2<HeavyCompare, Integer>>() {
				@Override
				public Tuple2<HeavyCompare, Integer> join(
					Tuple2<HeavyCompare, Integer> first,
					Tuple2<HeavyCompare, Integer> second) throws Exception {
					throw new Exception("Job should be canceled in sort-merge phase, never run here ...");
				}
			})
			.output(new DiscardingOutputFormat<Tuple2<HeavyCompare, Integer>>());

	runAndCancelJob(env.createProgramPlan(), 30 * 1000, 60 * 1000);
}
Example #4
Source File: JoinCancelingITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void executeTaskWithGenerator(
		JoinFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> joiner,
		int keys, int vals, int msecsTillCanceling, int maxTimeTillCanceled) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple2<Integer, Integer>> input1 = env.createInput(new UniformIntTupleGeneratorInputFormat(keys, vals));
	DataSet<Tuple2<Integer, Integer>> input2 = env.createInput(new UniformIntTupleGeneratorInputFormat(keys, vals));

	input1.join(input2, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE)
			.where(0)
			.equalTo(0)
			.with(joiner)
			.output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());

	env.setParallelism(PARALLELISM);

	runAndCancelJob(env.createProgramPlan(), msecsTillCanceling, maxTimeTillCanceled);
}
Example #5
Source File: JoinCancelingITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void executeTaskWithGenerator(
		JoinFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> joiner,
		int keys, int vals, int msecsTillCanceling, int maxTimeTillCanceled) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple2<Integer, Integer>> input1 = env.createInput(new UniformIntTupleGeneratorInputFormat(keys, vals));
	DataSet<Tuple2<Integer, Integer>> input2 = env.createInput(new UniformIntTupleGeneratorInputFormat(keys, vals));

	input1.join(input2, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE)
			.where(0)
			.equalTo(0)
			.with(joiner)
			.output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());

	env.setParallelism(PARALLELISM);

	runAndCancelJob(env.createProgramPlan(), msecsTillCanceling, maxTimeTillCanceled);
}
Example #6
Source File: JoinCancelingITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCancelSortMatchWhileDoingHeavySorting() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	HeavyCompareGeneratorInputFormat input = new HeavyCompareGeneratorInputFormat(100);
	DataSet<Tuple2<HeavyCompare, Integer>> input1 = env.createInput(input);
	DataSet<Tuple2<HeavyCompare, Integer>> input2 = env.createInput(input);

	input1.join(input2, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE)
			.where(0)
			.equalTo(0)
			.with(new JoinFunction<Tuple2<HeavyCompare, Integer>, Tuple2<HeavyCompare, Integer>, Tuple2<HeavyCompare, Integer>>() {
				@Override
				public Tuple2<HeavyCompare, Integer> join(
					Tuple2<HeavyCompare, Integer> first,
					Tuple2<HeavyCompare, Integer> second) throws Exception {
					throw new Exception("Job should be canceled in sort-merge phase, never run here ...");
				}
			})
			.output(new DiscardingOutputFormat<Tuple2<HeavyCompare, Integer>>());

	runAndCancelJob(env.createProgramPlan(), 30 * 1000, 60 * 1000);
}
Example #7
Source File: JoinOperator.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
public JoinOperatorBase<?, ?, OUT, ?> build() {
	JoinOperatorBase<?, ?, OUT, ?> operator;
	if (joinType.isOuter()) {
		operator = new OuterJoinOperatorBase<>(
				udf,
				new BinaryOperatorInformation(input1Type, input2Type, resultType),
				this.keys1.computeLogicalKeyPositions(),
				this.keys2.computeLogicalKeyPositions(),
				this.name,
				getOuterJoinType());
	} else {
		operator = new InnerJoinOperatorBase<>(
				udf,
				new BinaryOperatorInformation(input1Type, input2Type, resultType),
				this.keys1.computeLogicalKeyPositions(),
				this.keys2.computeLogicalKeyPositions(),
				this.name);
	}

	operator.setFirstInput(input1);
	operator.setSecondInput(input2);
	operator.setParallelism(parallelism);
	operator.setCustomPartitioner(partitioner);
	operator.setJoinHint(joinHint);
	return operator;
}
Example #8
Source File: JoinOperator.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
public JoinOperatorBase<?, ?, OUT, ?> build() {
	JoinOperatorBase<?, ?, OUT, ?> operator;
	if (joinType.isOuter()) {
		operator = new OuterJoinOperatorBase<>(
				udf,
				new BinaryOperatorInformation(input1Type, input2Type, resultType),
				this.keys1.computeLogicalKeyPositions(),
				this.keys2.computeLogicalKeyPositions(),
				this.name,
				getOuterJoinType());
	} else {
		operator = new InnerJoinOperatorBase<>(
				udf,
				new BinaryOperatorInformation(input1Type, input2Type, resultType),
				this.keys1.computeLogicalKeyPositions(),
				this.keys2.computeLogicalKeyPositions(),
				this.name);
	}

	operator.setFirstInput(input1);
	operator.setSecondInput(input2);
	operator.setParallelism(parallelism);
	operator.setCustomPartitioner(partitioner);
	operator.setJoinHint(joinHint);
	return operator;
}
Example #9
Source File: JoinCancelingITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void executeTask(JoinFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> joiner, boolean slow, int parallelism) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple2<Integer, Integer>> input1 = env.createInput(new InfiniteIntegerTupleInputFormat(slow));
	DataSet<Tuple2<Integer, Integer>> input2 = env.createInput(new InfiniteIntegerTupleInputFormat(slow));

	input1.join(input2, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE)
			.where(0)
			.equalTo(0)
			.with(joiner)
			.output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());

	env.setParallelism(parallelism);

	runAndCancelJob(env.createProgramPlan(), 5 * 1000, 10 * 1000);
}
Example #10
Source File: JoinCancelingITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void executeTask(JoinFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> joiner, boolean slow, int parallelism) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple2<Integer, Integer>> input1 = env.createInput(new InfiniteIntegerTupleInputFormat(slow));
	DataSet<Tuple2<Integer, Integer>> input2 = env.createInput(new InfiniteIntegerTupleInputFormat(slow));

	input1.join(input2, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE)
			.where(0)
			.equalTo(0)
			.with(joiner)
			.output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());

	env.setParallelism(parallelism);

	runAndCancelJob(env.createProgramPlan(), 5 * 1000, 10 * 1000);
}
Example #11
Source File: JoinCancelingITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void executeTask(JoinFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> joiner, boolean slow, int parallelism) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple2<Integer, Integer>> input1 = env.createInput(new InfiniteIntegerTupleInputFormat(slow));
	DataSet<Tuple2<Integer, Integer>> input2 = env.createInput(new InfiniteIntegerTupleInputFormat(slow));

	input1.join(input2, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE)
			.where(0)
			.equalTo(0)
			.with(joiner)
			.output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());

	env.setParallelism(parallelism);

	runAndCancelJob(env.createProgramPlan(), 5 * 1000, 10 * 1000);
}
Example #12
Source File: JoinOperator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
public JoinOperatorBase<?, ?, OUT, ?> build() {
	JoinOperatorBase<?, ?, OUT, ?> operator;
	if (joinType.isOuter()) {
		operator = new OuterJoinOperatorBase<>(
				udf,
				new BinaryOperatorInformation(input1Type, input2Type, resultType),
				this.keys1.computeLogicalKeyPositions(),
				this.keys2.computeLogicalKeyPositions(),
				this.name,
				getOuterJoinType());
	} else {
		operator = new InnerJoinOperatorBase<>(
				udf,
				new BinaryOperatorInformation(input1Type, input2Type, resultType),
				this.keys1.computeLogicalKeyPositions(),
				this.keys2.computeLogicalKeyPositions(),
				this.name);
	}

	operator.setFirstInput(input1);
	operator.setSecondInput(input2);
	operator.setParallelism(parallelism);
	operator.setCustomPartitioner(partitioner);
	operator.setJoinHint(joinHint);
	return operator;
}
Example #13
Source File: TriangleListing.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// u, v, bitmask where u < v
	DataSet<Tuple3<K, K, ByteValue>> filteredByID = input
		.getEdges()
		.map(new OrderByID<>())
			.setParallelism(parallelism)
			.name("Order by ID")
		.groupBy(0, 1)
		.reduceGroup(new ReduceBitmask<>())
			.setParallelism(parallelism)
			.name("Flatten by ID");

	// u, v, (deg(u), deg(v))
	DataSet<Edge<K, Tuple3<EV, Degrees, Degrees>>> pairDegrees = input
		.run(new EdgeDegreesPair<K, VV, EV>()
			.setParallelism(parallelism));

	// u, v, bitmask where deg(u) < deg(v) or (deg(u) == deg(v) and u < v)
	DataSet<Tuple3<K, K, ByteValue>> filteredByDegree = pairDegrees
		.map(new OrderByDegree<>())
			.setParallelism(parallelism)
			.name("Order by degree")
		.groupBy(0, 1)
		.reduceGroup(new ReduceBitmask<>())
			.setParallelism(parallelism)
			.name("Flatten by degree");

	// u, v, w, bitmask where (u, v) and (u, w) are edges in graph
	DataSet<Tuple4<K, K, K, ByteValue>> triplets = filteredByDegree
		.groupBy(0)
		.sortGroup(1, Order.ASCENDING)
		.reduceGroup(new GenerateTriplets<>())
			.name("Generate triplets");

	// u, v, w, bitmask where (u, v), (u, w), and (v, w) are edges in graph
	DataSet<Result<K>> triangles = triplets
		.join(filteredByID, JoinOperatorBase.JoinHint.REPARTITION_HASH_SECOND)
		.where(1, 2)
		.equalTo(0, 1)
		.with(new ProjectTriangles<>())
			.name("Triangle listing");

	if (permuteResults) {
		triangles = triangles
			.flatMap(new PermuteResult<>())
				.name("Permute triangle vertices");
	} else if (sortTriangleVertices.get()) {
		triangles = triangles
			.map(new SortTriangleVertices<>())
				.name("Sort triangle vertices");
	}

	return triangles;
}
Example #14
Source File: TriangleListing.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// u, v where u < v
	DataSet<Tuple2<K, K>> filteredByID = input
		.getEdges()
		.flatMap(new FilterByID<>())
			.setParallelism(parallelism)
			.name("Filter by ID");

	// u, v, (edge value, deg(u), deg(v))
	DataSet<Edge<K, Tuple3<EV, LongValue, LongValue>>> pairDegree = input
		.run(new EdgeDegreePair<K, VV, EV>()
			.setParallelism(parallelism));

	// u, v where deg(u) < deg(v) or (deg(u) == deg(v) and u < v)
	DataSet<Tuple2<K, K>> filteredByDegree = pairDegree
		.flatMap(new FilterByDegree<>())
			.setParallelism(parallelism)
			.name("Filter by degree");

	// u, v, w where (u, v) and (u, w) are edges in graph, v < w
	DataSet<Tuple3<K, K, K>> triplets = filteredByDegree
		.groupBy(0)
		.sortGroup(1, Order.ASCENDING)
		.reduceGroup(new GenerateTriplets<>())
			.name("Generate triplets");

	// u, v, w where (u, v), (u, w), and (v, w) are edges in graph, v < w
	DataSet<Result<K>> triangles = triplets
		.join(filteredByID, JoinOperatorBase.JoinHint.REPARTITION_HASH_SECOND)
		.where(1, 2)
		.equalTo(0, 1)
		.with(new ProjectTriangles<>())
			.name("Triangle listing");

	if (permuteResults) {
		triangles = triangles
			.flatMap(new PermuteResult<>())
				.name("Permute triangle vertices");
	} else if (sortTriangleVertices.get()) {
		triangles = triangles
			.map(new SortTriangleVertices<>())
				.name("Sort triangle vertices");
	}

	return triangles;
}
Example #15
Source File: TriangleListing.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// u, v where u < v
	DataSet<Tuple2<K, K>> filteredByID = input
		.getEdges()
		.flatMap(new FilterByID<>())
			.setParallelism(parallelism)
			.name("Filter by ID");

	// u, v, (edge value, deg(u), deg(v))
	DataSet<Edge<K, Tuple3<EV, LongValue, LongValue>>> pairDegree = input
		.run(new EdgeDegreePair<K, VV, EV>()
			.setParallelism(parallelism));

	// u, v where deg(u) < deg(v) or (deg(u) == deg(v) and u < v)
	DataSet<Tuple2<K, K>> filteredByDegree = pairDegree
		.flatMap(new FilterByDegree<>())
			.setParallelism(parallelism)
			.name("Filter by degree");

	// u, v, w where (u, v) and (u, w) are edges in graph, v < w
	DataSet<Tuple3<K, K, K>> triplets = filteredByDegree
		.groupBy(0)
		.sortGroup(1, Order.ASCENDING)
		.reduceGroup(new GenerateTriplets<>())
			.name("Generate triplets");

	// u, v, w where (u, v), (u, w), and (v, w) are edges in graph, v < w
	DataSet<Result<K>> triangles = triplets
		.join(filteredByID, JoinOperatorBase.JoinHint.REPARTITION_HASH_SECOND)
		.where(1, 2)
		.equalTo(0, 1)
		.with(new ProjectTriangles<>())
			.name("Triangle listing");

	if (permuteResults) {
		triangles = triangles
			.flatMap(new PermuteResult<>())
				.name("Permute triangle vertices");
	} else if (sortTriangleVertices.get()) {
		triangles = triangles
			.map(new SortTriangleVertices<>())
				.name("Sort triangle vertices");
	}

	return triangles;
}
Example #16
Source File: TriangleListing.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// u, v, bitmask where u < v
	DataSet<Tuple3<K, K, ByteValue>> filteredByID = input
		.getEdges()
		.map(new OrderByID<>())
			.setParallelism(parallelism)
			.name("Order by ID")
		.groupBy(0, 1)
		.reduceGroup(new ReduceBitmask<>())
			.setParallelism(parallelism)
			.name("Flatten by ID");

	// u, v, (deg(u), deg(v))
	DataSet<Edge<K, Tuple3<EV, Degrees, Degrees>>> pairDegrees = input
		.run(new EdgeDegreesPair<K, VV, EV>()
			.setParallelism(parallelism));

	// u, v, bitmask where deg(u) < deg(v) or (deg(u) == deg(v) and u < v)
	DataSet<Tuple3<K, K, ByteValue>> filteredByDegree = pairDegrees
		.map(new OrderByDegree<>())
			.setParallelism(parallelism)
			.name("Order by degree")
		.groupBy(0, 1)
		.reduceGroup(new ReduceBitmask<>())
			.setParallelism(parallelism)
			.name("Flatten by degree");

	// u, v, w, bitmask where (u, v) and (u, w) are edges in graph
	DataSet<Tuple4<K, K, K, ByteValue>> triplets = filteredByDegree
		.groupBy(0)
		.sortGroup(1, Order.ASCENDING)
		.reduceGroup(new GenerateTriplets<>())
			.name("Generate triplets");

	// u, v, w, bitmask where (u, v), (u, w), and (v, w) are edges in graph
	DataSet<Result<K>> triangles = triplets
		.join(filteredByID, JoinOperatorBase.JoinHint.REPARTITION_HASH_SECOND)
		.where(1, 2)
		.equalTo(0, 1)
		.with(new ProjectTriangles<>())
			.name("Triangle listing");

	if (permuteResults) {
		triangles = triangles
			.flatMap(new PermuteResult<>())
				.name("Permute triangle vertices");
	} else if (sortTriangleVertices.get()) {
		triangles = triangles
			.map(new SortTriangleVertices<>())
				.name("Sort triangle vertices");
	}

	return triangles;
}
Example #17
Source File: TriangleListing.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// u, v, bitmask where u < v
	DataSet<Tuple3<K, K, ByteValue>> filteredByID = input
		.getEdges()
		.map(new OrderByID<>())
			.setParallelism(parallelism)
			.name("Order by ID")
		.groupBy(0, 1)
		.reduceGroup(new ReduceBitmask<>())
			.setParallelism(parallelism)
			.name("Flatten by ID");

	// u, v, (deg(u), deg(v))
	DataSet<Edge<K, Tuple3<EV, Degrees, Degrees>>> pairDegrees = input
		.run(new EdgeDegreesPair<K, VV, EV>()
			.setParallelism(parallelism));

	// u, v, bitmask where deg(u) < deg(v) or (deg(u) == deg(v) and u < v)
	DataSet<Tuple3<K, K, ByteValue>> filteredByDegree = pairDegrees
		.map(new OrderByDegree<>())
			.setParallelism(parallelism)
			.name("Order by degree")
		.groupBy(0, 1)
		.reduceGroup(new ReduceBitmask<>())
			.setParallelism(parallelism)
			.name("Flatten by degree");

	// u, v, w, bitmask where (u, v) and (u, w) are edges in graph
	DataSet<Tuple4<K, K, K, ByteValue>> triplets = filteredByDegree
		.groupBy(0)
		.sortGroup(1, Order.ASCENDING)
		.reduceGroup(new GenerateTriplets<>())
			.name("Generate triplets");

	// u, v, w, bitmask where (u, v), (u, w), and (v, w) are edges in graph
	DataSet<Result<K>> triangles = triplets
		.join(filteredByID, JoinOperatorBase.JoinHint.REPARTITION_HASH_SECOND)
		.where(1, 2)
		.equalTo(0, 1)
		.with(new ProjectTriangles<>())
			.name("Triangle listing");

	if (permuteResults) {
		triangles = triangles
			.flatMap(new PermuteResult<>())
				.name("Permute triangle vertices");
	} else if (sortTriangleVertices.get()) {
		triangles = triangles
			.map(new SortTriangleVertices<>())
				.name("Sort triangle vertices");
	}

	return triangles;
}
Example #18
Source File: TriangleListing.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public DataSet<Result<K>> runInternal(Graph<K, VV, EV> input)
		throws Exception {
	// u, v where u < v
	DataSet<Tuple2<K, K>> filteredByID = input
		.getEdges()
		.flatMap(new FilterByID<>())
			.setParallelism(parallelism)
			.name("Filter by ID");

	// u, v, (edge value, deg(u), deg(v))
	DataSet<Edge<K, Tuple3<EV, LongValue, LongValue>>> pairDegree = input
		.run(new EdgeDegreePair<K, VV, EV>()
			.setParallelism(parallelism));

	// u, v where deg(u) < deg(v) or (deg(u) == deg(v) and u < v)
	DataSet<Tuple2<K, K>> filteredByDegree = pairDegree
		.flatMap(new FilterByDegree<>())
			.setParallelism(parallelism)
			.name("Filter by degree");

	// u, v, w where (u, v) and (u, w) are edges in graph, v < w
	DataSet<Tuple3<K, K, K>> triplets = filteredByDegree
		.groupBy(0)
		.sortGroup(1, Order.ASCENDING)
		.reduceGroup(new GenerateTriplets<>())
			.name("Generate triplets");

	// u, v, w where (u, v), (u, w), and (v, w) are edges in graph, v < w
	DataSet<Result<K>> triangles = triplets
		.join(filteredByID, JoinOperatorBase.JoinHint.REPARTITION_HASH_SECOND)
		.where(1, 2)
		.equalTo(0, 1)
		.with(new ProjectTriangles<>())
			.name("Triangle listing");

	if (permuteResults) {
		triangles = triangles
			.flatMap(new PermuteResult<>())
				.name("Permute triangle vertices");
	} else if (sortTriangleVertices.get()) {
		triangles = triangles
			.map(new SortTriangleVertices<>())
				.name("Sort triangle vertices");
	}

	return triangles;
}