org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner Java Examples

The following examples show how to use org.apache.flink.streaming.runtime.partitioner.ShufflePartitioner. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DataStreamTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testChannelSelectors() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<Long> src = env.generateSequence(0, 0);

	DataStream<Long> broadcast = src.broadcast();
	DataStreamSink<Long> broadcastSink = broadcast.print();
	StreamPartitioner<?> broadcastPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					broadcastSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);

	DataStream<Long> shuffle = src.shuffle();
	DataStreamSink<Long> shuffleSink = shuffle.print();
	StreamPartitioner<?> shufflePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					shuffleSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(shufflePartitioner instanceof ShufflePartitioner);

	DataStream<Long> forward = src.forward();
	DataStreamSink<Long> forwardSink = forward.print();
	StreamPartitioner<?> forwardPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					forwardSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(forwardPartitioner instanceof ForwardPartitioner);

	DataStream<Long> rebalance = src.rebalance();
	DataStreamSink<Long> rebalanceSink = rebalance.print();
	StreamPartitioner<?> rebalancePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(rebalancePartitioner instanceof RebalancePartitioner);

	DataStream<Long> global = src.global();
	DataStreamSink<Long> globalSink = global.print();
	StreamPartitioner<?> globalPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					globalSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(globalPartitioner instanceof GlobalPartitioner);
}
Example #2
Source File: StreamGraphGeneratorTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Verifies that partitioning, output selector, selected names are correctly set in the
 * StreamGraph when they are intermixed.
 */
@Test
public void testVirtualTransformations() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	// verify that only the partitioning that was set last is used
	DataStream<Integer> broadcastMap = rebalanceMap
			.forward()
			.global()
			.broadcast()
			.map(new NoOpIntMap());

	broadcastMap.addSink(new DiscardingSink<>());

	// verify that partitioning is preserved across union and split/select
	EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector3 = new EvenOddOutputSelector();

	DataStream<Integer> map1Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map1 = map1Operator
			.broadcast()
			.split(selector1)
			.select("even");

	DataStream<Integer> map2Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = map2Operator
			.split(selector2)
			.select("odd")
			.global();

	DataStream<Integer> map3Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = map3Operator
			.global()
			.split(selector3)
			.select("even")
			.shuffle();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// rebalanceMap
	assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);

	// verify that only last partitioning takes precedence
	assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());

	// verify that partitioning in unions is preserved and that it works across split/select
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));

	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));

	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}
Example #3
Source File: DataStreamTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testChannelSelectors() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<Long> src = env.generateSequence(0, 0);

	DataStream<Long> broadcast = src.broadcast();
	DataStreamSink<Long> broadcastSink = broadcast.print();
	StreamPartitioner<?> broadcastPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					broadcastSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);

	DataStream<Long> shuffle = src.shuffle();
	DataStreamSink<Long> shuffleSink = shuffle.print();
	StreamPartitioner<?> shufflePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					shuffleSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(shufflePartitioner instanceof ShufflePartitioner);

	DataStream<Long> forward = src.forward();
	DataStreamSink<Long> forwardSink = forward.print();
	StreamPartitioner<?> forwardPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					forwardSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(forwardPartitioner instanceof ForwardPartitioner);

	DataStream<Long> rebalance = src.rebalance();
	DataStreamSink<Long> rebalanceSink = rebalance.print();
	StreamPartitioner<?> rebalancePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(rebalancePartitioner instanceof RebalancePartitioner);

	DataStream<Long> global = src.global();
	DataStreamSink<Long> globalSink = global.print();
	StreamPartitioner<?> globalPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					globalSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(globalPartitioner instanceof GlobalPartitioner);
}
Example #4
Source File: StreamGraphGeneratorTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Verifies that partitioning, output selector, selected names are correctly set in the
 * StreamGraph when they are intermixed.
 */
@Test
public void testVirtualTransformations() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	// verify that only the partitioning that was set last is used
	DataStream<Integer> broadcastMap = rebalanceMap
			.forward()
			.global()
			.broadcast()
			.map(new NoOpIntMap());

	broadcastMap.addSink(new DiscardingSink<>());

	// verify that partitioning is preserved across union and split/select
	EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector3 = new EvenOddOutputSelector();

	DataStream<Integer> map1Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map1 = map1Operator
			.broadcast()
			.split(selector1)
			.select("even");

	DataStream<Integer> map2Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = map2Operator
			.split(selector2)
			.select("odd")
			.global();

	DataStream<Integer> map3Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = map3Operator
			.global()
			.split(selector3)
			.select("even")
			.shuffle();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// rebalanceMap
	assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);

	// verify that only last partitioning takes precedence
	assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());

	// verify that partitioning in unions is preserved and that it works across split/select
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));

	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));

	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}
Example #5
Source File: DataStreamTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testChannelSelectors() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<Long> src = env.generateSequence(0, 0);

	DataStream<Long> broadcast = src.broadcast();
	DataStreamSink<Long> broadcastSink = broadcast.print();
	StreamPartitioner<?> broadcastPartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					broadcastSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);

	DataStream<Long> shuffle = src.shuffle();
	DataStreamSink<Long> shuffleSink = shuffle.print();
	StreamPartitioner<?> shufflePartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					shuffleSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(shufflePartitioner instanceof ShufflePartitioner);

	DataStream<Long> forward = src.forward();
	DataStreamSink<Long> forwardSink = forward.print();
	StreamPartitioner<?> forwardPartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					forwardSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(forwardPartitioner instanceof ForwardPartitioner);

	DataStream<Long> rebalance = src.rebalance();
	DataStreamSink<Long> rebalanceSink = rebalance.print();
	StreamPartitioner<?> rebalancePartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(rebalancePartitioner instanceof RebalancePartitioner);

	DataStream<Long> global = src.global();
	DataStreamSink<Long> globalSink = global.print();
	StreamPartitioner<?> globalPartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					globalSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(globalPartitioner instanceof GlobalPartitioner);
}
Example #6
Source File: StreamGraphGeneratorTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Verifies that partitioning, output selector, selected names are correctly set in the
 * StreamGraph when they are intermixed.
 */
@Test
public void testVirtualTransformations() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	// verify that only the partitioning that was set last is used
	DataStream<Integer> broadcastMap = rebalanceMap
			.forward()
			.global()
			.broadcast()
			.map(new NoOpIntMap());

	broadcastMap.addSink(new DiscardingSink<>());

	// verify that partitioning is preserved across union and split/select
	EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector3 = new EvenOddOutputSelector();

	DataStream<Integer> map1Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map1 = map1Operator
			.broadcast()
			.split(selector1)
			.select("even");

	DataStream<Integer> map2Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = map2Operator
			.split(selector2)
			.select("odd")
			.global();

	DataStream<Integer> map3Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = map3Operator
			.global()
			.split(selector3)
			.select("even")
			.shuffle();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// rebalanceMap
	assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);

	// verify that only last partitioning takes precedence
	assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());

	// verify that partitioning in unions is preserved and that it works across split/select
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));

	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));

	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}
Example #7
Source File: DataStream.java    From Flink-CEPplus with Apache License 2.0 2 votes vote down vote up 
/**
 * Sets the partitioning of the {@link DataStream} so that the output elements
 * are shuffled uniformly randomly to the next operation.
 *
 * @return The DataStream with shuffle partitioning set.
 */
@PublicEvolving
public DataStream<T> shuffle() {
	return setConnectionType(new ShufflePartitioner<T>());
}
Example #8
Source File: DataStream.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Sets the partitioning of the {@link DataStream} so that the output elements
 * are shuffled uniformly randomly to the next operation.
 *
 * @return The DataStream with shuffle partitioning set.
 */
@PublicEvolving
public DataStream<T> shuffle() {
	return setConnectionType(new ShufflePartitioner<T>());
}
Example #9
Source File: DataStream.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Sets the partitioning of the {@link DataStream} so that the output elements
 * are shuffled uniformly randomly to the next operation.
 *
 * @return The DataStream with shuffle partitioning set.
 */
@PublicEvolving
public DataStream<T> shuffle() {
	return setConnectionType(new ShufflePartitioner<T>());
}