org.apache.flink.streaming.runtime.partitioner.RebalancePartitioner Java Examples
The following examples show how to use
org.apache.flink.streaming.runtime.partitioner.RebalancePartitioner.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: StreamingJobGraphGeneratorTest.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Create a StreamGraph as below.
*
* <p>source1 --(rebalance & pipelined)--> Map1
*
* <p>source2 --(rebalance & blocking)--> Map2
*/
private StreamGraph createStreamGraphForSlotSharingTest() {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
final DataStream<Integer> source1 = env.fromElements(1, 2, 3).name("source1");
source1.rebalance().map(v -> v).name("map1");
final DataStream<Integer> source2 = env.fromElements(4, 5, 6).name("source2");
final DataStream<Integer> partitioned = new DataStream<>(env, new PartitionTransformation<>(
source2.getTransformation(), new RebalancePartitioner<>(), ShuffleMode.BATCH));
partitioned.map(v -> v).name("map2");
return env.getStreamGraph();
}
Example #2
| Source File: DataStreamTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testChannelSelectors() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource<Long> src = env.generateSequence(0, 0);
DataStream<Long> broadcast = src.broadcast();
DataStreamSink<Long> broadcastSink = broadcast.print();
StreamPartitioner<?> broadcastPartitioner =
env.getStreamGraph().getStreamEdges(src.getId(),
broadcastSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);
DataStream<Long> shuffle = src.shuffle();
DataStreamSink<Long> shuffleSink = shuffle.print();
StreamPartitioner<?> shufflePartitioner =
env.getStreamGraph().getStreamEdges(src.getId(),
shuffleSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(shufflePartitioner instanceof ShufflePartitioner);
DataStream<Long> forward = src.forward();
DataStreamSink<Long> forwardSink = forward.print();
StreamPartitioner<?> forwardPartitioner =
env.getStreamGraph().getStreamEdges(src.getId(),
forwardSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(forwardPartitioner instanceof ForwardPartitioner);
DataStream<Long> rebalance = src.rebalance();
DataStreamSink<Long> rebalanceSink = rebalance.print();
StreamPartitioner<?> rebalancePartitioner =
env.getStreamGraph().getStreamEdges(src.getId(),
rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(rebalancePartitioner instanceof RebalancePartitioner);
DataStream<Long> global = src.global();
DataStreamSink<Long> globalSink = global.print();
StreamPartitioner<?> globalPartitioner =
env.getStreamGraph().getStreamEdges(src.getId(),
globalSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(globalPartitioner instanceof GlobalPartitioner);
}
Example #3
| Source File: StreamGraphGeneratorTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
/**
* This tests whether virtual Transformations behave correctly.
*
* <p>Verifies that partitioning, output selector, selected names are correctly set in the
* StreamGraph when they are intermixed.
*/
@Test
public void testVirtualTransformations() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStream<Integer> source = env.fromElements(1, 10);
DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());
// verify that only the partitioning that was set last is used
DataStream<Integer> broadcastMap = rebalanceMap
.forward()
.global()
.broadcast()
.map(new NoOpIntMap());
broadcastMap.addSink(new DiscardingSink<>());
// verify that partitioning is preserved across union and split/select
EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
EvenOddOutputSelector selector3 = new EvenOddOutputSelector();
DataStream<Integer> map1Operator = rebalanceMap
.map(new NoOpIntMap());
DataStream<Integer> map1 = map1Operator
.broadcast()
.split(selector1)
.select("even");
DataStream<Integer> map2Operator = rebalanceMap
.map(new NoOpIntMap());
DataStream<Integer> map2 = map2Operator
.split(selector2)
.select("odd")
.global();
DataStream<Integer> map3Operator = rebalanceMap
.map(new NoOpIntMap());
DataStream<Integer> map3 = map3Operator
.global()
.split(selector3)
.select("even")
.shuffle();
SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
.map(new NoOpIntMap());
unionedMap.addSink(new DiscardingSink<>());
StreamGraph graph = env.getStreamGraph();
// rebalanceMap
assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);
// verify that only last partitioning takes precedence
assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());
// verify that partitioning in unions is preserved and that it works across split/select
assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));
assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));
assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}
Example #4
| Source File: DataStreamTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testChannelSelectors() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource<Long> src = env.generateSequence(0, 0);
DataStream<Long> broadcast = src.broadcast();
DataStreamSink<Long> broadcastSink = broadcast.print();
StreamPartitioner<?> broadcastPartitioner =
env.getStreamGraph().getStreamEdges(src.getId(),
broadcastSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);
DataStream<Long> shuffle = src.shuffle();
DataStreamSink<Long> shuffleSink = shuffle.print();
StreamPartitioner<?> shufflePartitioner =
env.getStreamGraph().getStreamEdges(src.getId(),
shuffleSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(shufflePartitioner instanceof ShufflePartitioner);
DataStream<Long> forward = src.forward();
DataStreamSink<Long> forwardSink = forward.print();
StreamPartitioner<?> forwardPartitioner =
env.getStreamGraph().getStreamEdges(src.getId(),
forwardSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(forwardPartitioner instanceof ForwardPartitioner);
DataStream<Long> rebalance = src.rebalance();
DataStreamSink<Long> rebalanceSink = rebalance.print();
StreamPartitioner<?> rebalancePartitioner =
env.getStreamGraph().getStreamEdges(src.getId(),
rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(rebalancePartitioner instanceof RebalancePartitioner);
DataStream<Long> global = src.global();
DataStreamSink<Long> globalSink = global.print();
StreamPartitioner<?> globalPartitioner =
env.getStreamGraph().getStreamEdges(src.getId(),
globalSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(globalPartitioner instanceof GlobalPartitioner);
}
Example #5
| Source File: StreamGraphGeneratorTest.java From flink with Apache License 2.0 | 4 votes |
|
/**
* This tests whether virtual Transformations behave correctly.
*
* <p>Verifies that partitioning, output selector, selected names are correctly set in the
* StreamGraph when they are intermixed.
*/
@Test
public void testVirtualTransformations() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStream<Integer> source = env.fromElements(1, 10);
DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());
// verify that only the partitioning that was set last is used
DataStream<Integer> broadcastMap = rebalanceMap
.forward()
.global()
.broadcast()
.map(new NoOpIntMap());
broadcastMap.addSink(new DiscardingSink<>());
// verify that partitioning is preserved across union and split/select
EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
EvenOddOutputSelector selector3 = new EvenOddOutputSelector();
DataStream<Integer> map1Operator = rebalanceMap
.map(new NoOpIntMap());
DataStream<Integer> map1 = map1Operator
.broadcast()
.split(selector1)
.select("even");
DataStream<Integer> map2Operator = rebalanceMap
.map(new NoOpIntMap());
DataStream<Integer> map2 = map2Operator
.split(selector2)
.select("odd")
.global();
DataStream<Integer> map3Operator = rebalanceMap
.map(new NoOpIntMap());
DataStream<Integer> map3 = map3Operator
.global()
.split(selector3)
.select("even")
.shuffle();
SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
.map(new NoOpIntMap());
unionedMap.addSink(new DiscardingSink<>());
StreamGraph graph = env.getStreamGraph();
// rebalanceMap
assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);
// verify that only last partitioning takes precedence
assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());
// verify that partitioning in unions is preserved and that it works across split/select
assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));
assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));
assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}
Example #6
| Source File: DataStreamTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testChannelSelectors() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource<Long> src = env.generateSequence(0, 0);
DataStream<Long> broadcast = src.broadcast();
DataStreamSink<Long> broadcastSink = broadcast.print();
StreamPartitioner<?> broadcastPartitioner =
getStreamGraph(env).getStreamEdges(src.getId(),
broadcastSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);
DataStream<Long> shuffle = src.shuffle();
DataStreamSink<Long> shuffleSink = shuffle.print();
StreamPartitioner<?> shufflePartitioner =
getStreamGraph(env).getStreamEdges(src.getId(),
shuffleSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(shufflePartitioner instanceof ShufflePartitioner);
DataStream<Long> forward = src.forward();
DataStreamSink<Long> forwardSink = forward.print();
StreamPartitioner<?> forwardPartitioner =
getStreamGraph(env).getStreamEdges(src.getId(),
forwardSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(forwardPartitioner instanceof ForwardPartitioner);
DataStream<Long> rebalance = src.rebalance();
DataStreamSink<Long> rebalanceSink = rebalance.print();
StreamPartitioner<?> rebalancePartitioner =
getStreamGraph(env).getStreamEdges(src.getId(),
rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(rebalancePartitioner instanceof RebalancePartitioner);
DataStream<Long> global = src.global();
DataStreamSink<Long> globalSink = global.print();
StreamPartitioner<?> globalPartitioner =
getStreamGraph(env).getStreamEdges(src.getId(),
globalSink.getTransformation().getId()).get(0).getPartitioner();
assertTrue(globalPartitioner instanceof GlobalPartitioner);
}
Example #7
| Source File: StreamGraphGeneratorTest.java From flink with Apache License 2.0 | 4 votes |
|
/**
* This tests whether virtual Transformations behave correctly.
*
* <p>Verifies that partitioning, output selector, selected names are correctly set in the
* StreamGraph when they are intermixed.
*/
@Test
public void testVirtualTransformations() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStream<Integer> source = env.fromElements(1, 10);
DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());
// verify that only the partitioning that was set last is used
DataStream<Integer> broadcastMap = rebalanceMap
.forward()
.global()
.broadcast()
.map(new NoOpIntMap());
broadcastMap.addSink(new DiscardingSink<>());
// verify that partitioning is preserved across union and split/select
EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
EvenOddOutputSelector selector3 = new EvenOddOutputSelector();
DataStream<Integer> map1Operator = rebalanceMap
.map(new NoOpIntMap());
DataStream<Integer> map1 = map1Operator
.broadcast()
.split(selector1)
.select("even");
DataStream<Integer> map2Operator = rebalanceMap
.map(new NoOpIntMap());
DataStream<Integer> map2 = map2Operator
.split(selector2)
.select("odd")
.global();
DataStream<Integer> map3Operator = rebalanceMap
.map(new NoOpIntMap());
DataStream<Integer> map3 = map3Operator
.global()
.split(selector3)
.select("even")
.shuffle();
SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
.map(new NoOpIntMap());
unionedMap.addSink(new DiscardingSink<>());
StreamGraph graph = env.getStreamGraph();
// rebalanceMap
assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);
// verify that only last partitioning takes precedence
assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());
// verify that partitioning in unions is preserved and that it works across split/select
assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));
assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));
assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}
Example #8
| Source File: DataStream.java From Flink-CEPplus with Apache License 2.0 | 2 votes |
|
/**
* Sets the partitioning of the {@link DataStream} so that the output elements
* are distributed evenly to instances of the next operation in a round-robin
* fashion.
*
* @return The DataStream with rebalance partitioning set.
*/
public DataStream<T> rebalance() {
return setConnectionType(new RebalancePartitioner<T>());
}
Example #9
| Source File: DataStream.java From flink with Apache License 2.0 | 2 votes |
|
/**
* Sets the partitioning of the {@link DataStream} so that the output elements
* are distributed evenly to instances of the next operation in a round-robin
* fashion.
*
* @return The DataStream with rebalance partitioning set.
*/
public DataStream<T> rebalance() {
return setConnectionType(new RebalancePartitioner<T>());
}
Example #10
| Source File: DataStream.java From flink with Apache License 2.0 | 2 votes |
|
/**
* Sets the partitioning of the {@link DataStream} so that the output elements
* are distributed evenly to instances of the next operation in a round-robin
* fashion.
*
* @return The DataStream with rebalance partitioning set.
*/
public DataStream<T> rebalance() {
return setConnectionType(new RebalancePartitioner<T>());
}
