org.apache.flink.runtime.operators.util.LocalStrategy Java Examples
The following examples show how to use
org.apache.flink.runtime.operators.util.LocalStrategy.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: BinaryUnionReplacer.java From flink with Apache License 2.0 | 6 votes |
|
public void collect(Channel in, List<Channel> inputs) {
if (in.getSource() instanceof NAryUnionPlanNode) {
// sanity check
if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
}
if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
}
inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
} else {
// is not a collapsed union node, so we take the channel directly
inputs.add(in);
}
}
Example #2
| Source File: RelationalQueryCompilerTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
private boolean checkRepartitionMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) {
if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) {
// driver keys
Assert.assertEquals(set0, join.getKeysForInput1());
Assert.assertEquals(set0, join.getKeysForInput2());
// local strategies
Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy());
Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy());
Assert.assertEquals(LocalStrategy.NONE, reducer.getInput().getLocalStrategy());
// local strategy keys
Assert.assertEquals(set01, join.getInput1().getLocalStrategyKeys());
Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys());
Assert.assertTrue(join.getInput1().getLocalStrategySortOrder()[0] == join.getInput2().getLocalStrategySortOrder()[0]);
Assert.assertEquals(set01, reducer.getKeys(0));
Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), reducer.getSortOrders(0)));
return true;
} else {
return false;
}
}
Example #3
| Source File: RelationalQueryCompilerTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
private boolean checkBroadcastMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) {
if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) {
// driver keys
Assert.assertEquals(set0, join.getKeysForInput1());
Assert.assertEquals(set0, join.getKeysForInput2());
// local strategies
Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy());
Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy());
Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy());
// local strategy keys
Assert.assertEquals(set0, join.getInput1().getLocalStrategyKeys());
Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys());
Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), join.getInput2().getLocalStrategySortOrder()));
Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys());
Assert.assertEquals(set01, reducer.getKeys(0));
Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0)));
return true;
} else {
return false;
}
}
Example #4
| Source File: RelationalQueryCompilerTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
private boolean checkHashJoinStrategies(DualInputPlanNode join, SingleInputPlanNode reducer, boolean buildFirst) {
if ((buildFirst && DriverStrategy.HYBRIDHASH_BUILD_FIRST == join.getDriverStrategy()) ||
(!buildFirst && DriverStrategy.HYBRIDHASH_BUILD_SECOND == join.getDriverStrategy())) {
// driver keys
Assert.assertEquals(set0, join.getKeysForInput1());
Assert.assertEquals(set0, join.getKeysForInput2());
// local strategies
Assert.assertEquals(LocalStrategy.NONE, join.getInput1().getLocalStrategy());
Assert.assertEquals(LocalStrategy.NONE, join.getInput2().getLocalStrategy());
Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy());
// local strategy keys
Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys());
Assert.assertEquals(set01, reducer.getKeys(0));
Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0)));
return true;
} else {
return false;
}
}
Example #5
| Source File: BinaryUnionReplacer.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
public void collect(Channel in, List<Channel> inputs) {
if (in.getSource() instanceof NAryUnionPlanNode) {
// sanity check
if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
}
if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
}
inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
} else {
// is not a collapsed union node, so we take the channel directly
inputs.add(in);
}
}
Example #6
| Source File: BinaryUnionReplacer.java From flink with Apache License 2.0 | 6 votes |
|
public void collect(Channel in, List<Channel> inputs) {
if (in.getSource() instanceof NAryUnionPlanNode) {
// sanity check
if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
}
if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
}
inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
} else {
// is not a collapsed union node, so we take the channel directly
inputs.add(in);
}
}
Example #7
| Source File: RequestedLocalProperties.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
/**
* Parametrizes the local strategy fields of a channel such that the channel produces the desired local properties.
*
* @param channel The channel to parametrize.
*/
public void parameterizeChannel(Channel channel) {
LocalProperties current = channel.getLocalProperties();
if (isMetBy(current)) {
// we are met, all is good
channel.setLocalStrategy(LocalStrategy.NONE);
}
else if (this.ordering != null) {
channel.setLocalStrategy(LocalStrategy.SORT, this.ordering.getInvolvedIndexes(), this.ordering.getFieldSortDirections());
}
else if (this.groupedFields != null) {
boolean[] dirs = new boolean[this.groupedFields.size()];
Arrays.fill(dirs, true);
channel.setLocalStrategy(LocalStrategy.SORT, Utils.createOrderedFromSet(this.groupedFields), dirs);
}
else {
channel.setLocalStrategy(LocalStrategy.NONE);
}
}
Example #8
| Source File: RequestedLocalProperties.java From flink with Apache License 2.0 | 6 votes |
|
/**
* Parametrizes the local strategy fields of a channel such that the channel produces the desired local properties.
*
* @param channel The channel to parametrize.
*/
public void parameterizeChannel(Channel channel) {
LocalProperties current = channel.getLocalProperties();
if (isMetBy(current)) {
// we are met, all is good
channel.setLocalStrategy(LocalStrategy.NONE);
}
else if (this.ordering != null) {
channel.setLocalStrategy(LocalStrategy.SORT, this.ordering.getInvolvedIndexes(), this.ordering.getFieldSortDirections());
}
else if (this.groupedFields != null) {
boolean[] dirs = new boolean[this.groupedFields.size()];
Arrays.fill(dirs, true);
channel.setLocalStrategy(LocalStrategy.SORT, Utils.createOrderedFromSet(this.groupedFields), dirs);
}
else {
channel.setLocalStrategy(LocalStrategy.NONE);
}
}
Example #9
| Source File: RequestedLocalProperties.java From flink with Apache License 2.0 | 6 votes |
|
/**
* Parametrizes the local strategy fields of a channel such that the channel produces the desired local properties.
*
* @param channel The channel to parametrize.
*/
public void parameterizeChannel(Channel channel) {
LocalProperties current = channel.getLocalProperties();
if (isMetBy(current)) {
// we are met, all is good
channel.setLocalStrategy(LocalStrategy.NONE);
}
else if (this.ordering != null) {
channel.setLocalStrategy(LocalStrategy.SORT, this.ordering.getInvolvedIndexes(), this.ordering.getFieldSortDirections());
}
else if (this.groupedFields != null) {
boolean[] dirs = new boolean[this.groupedFields.size()];
Arrays.fill(dirs, true);
channel.setLocalStrategy(LocalStrategy.SORT, Utils.createOrderedFromSet(this.groupedFields), dirs);
}
else {
channel.setLocalStrategy(LocalStrategy.NONE);
}
}
Example #10
| Source File: RelationalQueryCompilerTest.java From flink with Apache License 2.0 | 6 votes |
|
private boolean checkRepartitionMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) {
if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) {
// driver keys
Assert.assertEquals(set0, join.getKeysForInput1());
Assert.assertEquals(set0, join.getKeysForInput2());
// local strategies
Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy());
Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy());
Assert.assertEquals(LocalStrategy.NONE, reducer.getInput().getLocalStrategy());
// local strategy keys
Assert.assertEquals(set01, join.getInput1().getLocalStrategyKeys());
Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys());
Assert.assertTrue(join.getInput1().getLocalStrategySortOrder()[0] == join.getInput2().getLocalStrategySortOrder()[0]);
Assert.assertEquals(set01, reducer.getKeys(0));
Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), reducer.getSortOrders(0)));
return true;
} else {
return false;
}
}
Example #11
| Source File: RelationalQueryCompilerTest.java From flink with Apache License 2.0 | 6 votes |
|
private boolean checkBroadcastMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) {
if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) {
// driver keys
Assert.assertEquals(set0, join.getKeysForInput1());
Assert.assertEquals(set0, join.getKeysForInput2());
// local strategies
Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy());
Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy());
Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy());
// local strategy keys
Assert.assertEquals(set0, join.getInput1().getLocalStrategyKeys());
Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys());
Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), join.getInput2().getLocalStrategySortOrder()));
Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys());
Assert.assertEquals(set01, reducer.getKeys(0));
Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0)));
return true;
} else {
return false;
}
}
Example #12
| Source File: RelationalQueryCompilerTest.java From flink with Apache License 2.0 | 6 votes |
|
private boolean checkHashJoinStrategies(DualInputPlanNode join, SingleInputPlanNode reducer, boolean buildFirst) {
if ((buildFirst && DriverStrategy.HYBRIDHASH_BUILD_FIRST == join.getDriverStrategy()) ||
(!buildFirst && DriverStrategy.HYBRIDHASH_BUILD_SECOND == join.getDriverStrategy())) {
// driver keys
Assert.assertEquals(set0, join.getKeysForInput1());
Assert.assertEquals(set0, join.getKeysForInput2());
// local strategies
Assert.assertEquals(LocalStrategy.NONE, join.getInput1().getLocalStrategy());
Assert.assertEquals(LocalStrategy.NONE, join.getInput2().getLocalStrategy());
Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy());
// local strategy keys
Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys());
Assert.assertEquals(set01, reducer.getKeys(0));
Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0)));
return true;
} else {
return false;
}
}
Example #13
| Source File: RelationalQueryCompilerTest.java From flink with Apache License 2.0 | 6 votes |
|
private boolean checkRepartitionMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) {
if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) {
// driver keys
Assert.assertEquals(set0, join.getKeysForInput1());
Assert.assertEquals(set0, join.getKeysForInput2());
// local strategies
Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy());
Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy());
Assert.assertEquals(LocalStrategy.NONE, reducer.getInput().getLocalStrategy());
// local strategy keys
Assert.assertEquals(set01, join.getInput1().getLocalStrategyKeys());
Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys());
Assert.assertTrue(join.getInput1().getLocalStrategySortOrder()[0] == join.getInput2().getLocalStrategySortOrder()[0]);
Assert.assertEquals(set01, reducer.getKeys(0));
Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), reducer.getSortOrders(0)));
return true;
} else {
return false;
}
}
Example #14
| Source File: RelationalQueryCompilerTest.java From flink with Apache License 2.0 | 6 votes |
|
private boolean checkBroadcastMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) {
if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) {
// driver keys
Assert.assertEquals(set0, join.getKeysForInput1());
Assert.assertEquals(set0, join.getKeysForInput2());
// local strategies
Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy());
Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy());
Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy());
// local strategy keys
Assert.assertEquals(set0, join.getInput1().getLocalStrategyKeys());
Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys());
Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), join.getInput2().getLocalStrategySortOrder()));
Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys());
Assert.assertEquals(set01, reducer.getKeys(0));
Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0)));
return true;
} else {
return false;
}
}
Example #15
| Source File: RelationalQueryCompilerTest.java From flink with Apache License 2.0 | 6 votes |
|
private boolean checkHashJoinStrategies(DualInputPlanNode join, SingleInputPlanNode reducer, boolean buildFirst) {
if ((buildFirst && DriverStrategy.HYBRIDHASH_BUILD_FIRST == join.getDriverStrategy()) ||
(!buildFirst && DriverStrategy.HYBRIDHASH_BUILD_SECOND == join.getDriverStrategy())) {
// driver keys
Assert.assertEquals(set0, join.getKeysForInput1());
Assert.assertEquals(set0, join.getKeysForInput2());
// local strategies
Assert.assertEquals(LocalStrategy.NONE, join.getInput1().getLocalStrategy());
Assert.assertEquals(LocalStrategy.NONE, join.getInput2().getLocalStrategy());
Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy());
// local strategy keys
Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys());
Assert.assertEquals(set01, reducer.getKeys(0));
Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0)));
return true;
} else {
return false;
}
}
Example #16
| Source File: ParallelismChangeTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void checkPropertyHandlingWithDecreasingParallelism() {
final int p = DEFAULT_PARALLELISM;
// construct the plan
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(p);
env
.generateSequence(0, 1).setParallelism(p * 2)
.map(new IdentityMapper<Long>())
.withForwardedFields("*").setParallelism(p * 2).name("Map1")
.groupBy("*").reduceGroup(new IdentityGroupReducer<Long>())
.withForwardedFields("*").setParallelism(p * 2).name("Reduce1")
.map(new IdentityMapper<Long>())
.withForwardedFields("*").setParallelism(p).name("Map2")
.groupBy("*").reduceGroup(new IdentityGroupReducer<Long>())
.withForwardedFields("*").setParallelism(p).name("Reduce2")
.output(new DiscardingOutputFormat<Long>()).setParallelism(p).name("Sink");
Plan plan = env.createProgramPlan();
// submit the plan to the compiler
OptimizedPlan oPlan = compileNoStats(plan);
// check the optimized Plan
// when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
// because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
// mapper respectively reducer
SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
Assert.assertTrue("The no sorting local strategy.",
LocalStrategy.SORT == red2Node.getInput().getLocalStrategy() ||
LocalStrategy.SORT == map2Node.getInput().getLocalStrategy());
Assert.assertTrue("The no partitioning ship strategy.",
ShipStrategyType.PARTITION_HASH == red2Node.getInput().getShipStrategy() ||
ShipStrategyType.PARTITION_HASH == map2Node.getInput().getShipStrategy());
}
Example #17
| Source File: JsonMapper.java From flink with Apache License 2.0 | 5 votes |
|
public static String getLocalStrategyString(LocalStrategy localStrategy) {
if (localStrategy == null) {
return "(null)";
}
switch (localStrategy) {
case NONE:
return "(none)";
case SORT:
return "Sort";
case COMBININGSORT:
return "Sort (combining)";
default:
return localStrategy.name();
}
}
Example #18
| Source File: DataSinkTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
@SuppressWarnings("unchecked")
public void testFailingSortingDataSinkTask() {
int keyCnt = 100;
int valCnt = 20;
double memoryFraction = 1.0;
super.initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
super.addInput(new UniformRecordGenerator(keyCnt, valCnt, true), 0);
DataSinkTask<Record> testTask = new DataSinkTask<>(this.mockEnv);
Configuration stubParams = new Configuration();
// set sorting
super.getTaskConfig().setInputLocalStrategy(0, LocalStrategy.SORT);
super.getTaskConfig().setInputComparator(
new RecordComparatorFactory(new int[]{1}, ( new Class[]{IntValue.class})), 0);
super.getTaskConfig().setRelativeMemoryInput(0, memoryFraction);
super.getTaskConfig().setFilehandlesInput(0, 8);
super.getTaskConfig().setSpillingThresholdInput(0, 0.8f);
File tempTestFile = new File(tempFolder.getRoot(), UUID.randomUUID().toString());
super.registerFileOutputTask(MockFailingOutputFormat.class, tempTestFile.toURI().toString(), stubParams);
boolean stubFailed = false;
try {
testTask.invoke();
} catch (Exception e) {
stubFailed = true;
}
Assert.assertTrue("Function exception was not forwarded.", stubFailed);
// assert that temp file was removed
Assert.assertFalse("Temp output file has not been removed", tempTestFile.exists());
}
Example #19
| Source File: ParallelismChangeTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void checkPropertyHandlingWithDecreasingParallelism() {
final int p = DEFAULT_PARALLELISM;
// construct the plan
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(p);
env
.generateSequence(0, 1).setParallelism(p * 2)
.map(new IdentityMapper<Long>())
.withForwardedFields("*").setParallelism(p * 2).name("Map1")
.groupBy("*").reduceGroup(new IdentityGroupReducer<Long>())
.withForwardedFields("*").setParallelism(p * 2).name("Reduce1")
.map(new IdentityMapper<Long>())
.withForwardedFields("*").setParallelism(p).name("Map2")
.groupBy("*").reduceGroup(new IdentityGroupReducer<Long>())
.withForwardedFields("*").setParallelism(p).name("Reduce2")
.output(new DiscardingOutputFormat<Long>()).setParallelism(p).name("Sink");
Plan plan = env.createProgramPlan();
// submit the plan to the compiler
OptimizedPlan oPlan = compileNoStats(plan);
// check the optimized Plan
// when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
// because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
// mapper respectively reducer
SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
Assert.assertTrue("The no sorting local strategy.",
LocalStrategy.SORT == red2Node.getInput().getLocalStrategy() ||
LocalStrategy.SORT == map2Node.getInput().getLocalStrategy());
Assert.assertTrue("The no partitioning ship strategy.",
ShipStrategyType.PARTITION_HASH == red2Node.getInput().getShipStrategy() ||
ShipStrategyType.PARTITION_HASH == map2Node.getInput().getShipStrategy());
}
Example #20
| Source File: FeedbackPropertiesMatchTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testNoPartialSolutionFoundTwoInputOperator() {
try {
SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1");
SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
Channel toMap1 = new Channel(source1);
toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap1.setLocalStrategy(LocalStrategy.NONE);
SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
Channel toMap2 = new Channel(source2);
toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap2.setLocalStrategy(LocalStrategy.NONE);
SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
Channel toJoin1 = new Channel(map1);
Channel toJoin2 = new Channel(map2);
toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toJoin1.setLocalStrategy(LocalStrategy.NONE);
toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toJoin2.setLocalStrategy(LocalStrategy.NONE);
DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
assertEquals(NO_PARTIAL_SOLUTION, report);
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #21
| Source File: RelationalQueryCompilerTest.java From flink with Apache License 2.0 | 5 votes |
|
private void checkStandardStrategies(SingleInputPlanNode map, DualInputPlanNode join, SingleInputPlanNode combiner,
SingleInputPlanNode reducer, SinkPlanNode sink) {
// check ship strategies that are always fix
Assert.assertEquals(ShipStrategyType.FORWARD, map.getInput().getShipStrategy());
Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
// check the driver strategies that are always fix
Assert.assertEquals(DriverStrategy.FLAT_MAP, map.getDriverStrategy());
Assert.assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reducer.getDriverStrategy());
Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy());
if (combiner != null) {
Assert.assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy());
Assert.assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy());
}
}
Example #22
| Source File: DataSinkTaskTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
@SuppressWarnings("unchecked")
public void testFailingSortingDataSinkTask() {
int keyCnt = 100;
int valCnt = 20;
double memoryFraction = 1.0;
super.initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
super.addInput(new UniformRecordGenerator(keyCnt, valCnt, true), 0);
DataSinkTask<Record> testTask = new DataSinkTask<>(this.mockEnv);
Configuration stubParams = new Configuration();
super.getTaskConfig().setStubParameters(stubParams);
// set sorting
super.getTaskConfig().setInputLocalStrategy(0, LocalStrategy.SORT);
super.getTaskConfig().setInputComparator(
new RecordComparatorFactory(new int[]{1}, ( new Class[]{IntValue.class})), 0);
super.getTaskConfig().setRelativeMemoryInput(0, memoryFraction);
super.getTaskConfig().setFilehandlesInput(0, 8);
super.getTaskConfig().setSpillingThresholdInput(0, 0.8f);
File tempTestFile = new File(tempFolder.getRoot(), UUID.randomUUID().toString());
super.registerFileOutputTask(MockFailingOutputFormat.class, tempTestFile.toURI().toString());
boolean stubFailed = false;
try {
testTask.invoke();
} catch (Exception e) {
stubFailed = true;
}
Assert.assertTrue("Function exception was not forwarded.", stubFailed);
// assert that temp file was removed
Assert.assertFalse("Temp output file has not been removed", tempTestFile.exists());
}
Example #23
| Source File: FeedbackPropertiesMatchTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testNoPartialSolutionFoundSingleInputOnly() {
try {
SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
SourcePlanNode otherTarget = new SourcePlanNode(getSourceNode(), "Source");
Channel toMap1 = new Channel(target);
toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap1.setLocalStrategy(LocalStrategy.NONE);
SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
Channel toMap2 = new Channel(map1);
toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap2.setLocalStrategy(LocalStrategy.NONE);
SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
{
GlobalProperties gp = new GlobalProperties();
LocalProperties lp = new LocalProperties();
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(otherTarget, gp, lp);
assertTrue(report == NO_PARTIAL_SOLUTION);
}
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #24
| Source File: FeedbackPropertiesMatchTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testNoPartialSolutionFoundTwoInputOperator() {
try {
SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1");
SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
Channel toMap1 = new Channel(source1);
toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap1.setLocalStrategy(LocalStrategy.NONE);
SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
Channel toMap2 = new Channel(source2);
toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap2.setLocalStrategy(LocalStrategy.NONE);
SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
Channel toJoin1 = new Channel(map1);
Channel toJoin2 = new Channel(map2);
toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toJoin1.setLocalStrategy(LocalStrategy.NONE);
toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toJoin2.setLocalStrategy(LocalStrategy.NONE);
DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
assertEquals(NO_PARTIAL_SOLUTION, report);
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #25
| Source File: DataSinkTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
@SuppressWarnings("unchecked")
public void testFailingSortingDataSinkTask() {
int keyCnt = 100;
int valCnt = 20;
double memoryFraction = 1.0;
super.initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
super.addInput(new UniformRecordGenerator(keyCnt, valCnt, true), 0);
DataSinkTask<Record> testTask = new DataSinkTask<>(this.mockEnv);
Configuration stubParams = new Configuration();
// set sorting
super.getTaskConfig().setInputLocalStrategy(0, LocalStrategy.SORT);
super.getTaskConfig().setInputComparator(
new RecordComparatorFactory(new int[]{1}, ( new Class[]{IntValue.class})), 0);
super.getTaskConfig().setRelativeMemoryInput(0, memoryFraction);
super.getTaskConfig().setFilehandlesInput(0, 8);
super.getTaskConfig().setSpillingThresholdInput(0, 0.8f);
File tempTestFile = new File(tempFolder.getRoot(), UUID.randomUUID().toString());
super.registerFileOutputTask(MockFailingOutputFormat.class, tempTestFile.toURI().toString(), stubParams);
boolean stubFailed = false;
try {
testTask.invoke();
} catch (Exception e) {
stubFailed = true;
}
Assert.assertTrue("Function exception was not forwarded.", stubFailed);
// assert that temp file was removed
Assert.assertFalse("Temp output file has not been removed", tempTestFile.exists());
}
Example #26
| Source File: RelationalQueryCompilerTest.java From flink with Apache License 2.0 | 5 votes |
|
private void checkStandardStrategies(SingleInputPlanNode map, DualInputPlanNode join, SingleInputPlanNode combiner,
SingleInputPlanNode reducer, SinkPlanNode sink) {
// check ship strategies that are always fix
Assert.assertEquals(ShipStrategyType.FORWARD, map.getInput().getShipStrategy());
Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
// check the driver strategies that are always fix
Assert.assertEquals(DriverStrategy.FLAT_MAP, map.getDriverStrategy());
Assert.assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reducer.getDriverStrategy());
Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy());
if (combiner != null) {
Assert.assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy());
Assert.assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy());
}
}
Example #27
| Source File: FeedbackPropertiesMatchTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testNoPartialSolutionFoundTwoInputOperator() {
try {
SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1");
SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
Channel toMap1 = new Channel(source1);
toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap1.setLocalStrategy(LocalStrategy.NONE);
SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
Channel toMap2 = new Channel(source2);
toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap2.setLocalStrategy(LocalStrategy.NONE);
SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
Channel toJoin1 = new Channel(map1);
Channel toJoin2 = new Channel(map2);
toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toJoin1.setLocalStrategy(LocalStrategy.NONE);
toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toJoin2.setLocalStrategy(LocalStrategy.NONE);
DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
assertEquals(NO_PARTIAL_SOLUTION, report);
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #28
| Source File: RelationalQueryCompilerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private void checkStandardStrategies(SingleInputPlanNode map, DualInputPlanNode join, SingleInputPlanNode combiner,
SingleInputPlanNode reducer, SinkPlanNode sink) {
// check ship strategies that are always fix
Assert.assertEquals(ShipStrategyType.FORWARD, map.getInput().getShipStrategy());
Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
// check the driver strategies that are always fix
Assert.assertEquals(DriverStrategy.FLAT_MAP, map.getDriverStrategy());
Assert.assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reducer.getDriverStrategy());
Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy());
if (combiner != null) {
Assert.assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy());
Assert.assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy());
}
}
Example #29
| Source File: JsonMapper.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
public static String getLocalStrategyString(LocalStrategy localStrategy) {
if (localStrategy == null) {
return "(null)";
}
switch (localStrategy) {
case NONE:
return "(none)";
case SORT:
return "Sort";
case COMBININGSORT:
return "Sort (combining)";
default:
return localStrategy.name();
}
}
Example #30
| Source File: FeedbackPropertiesMatchTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testNoPartialSolutionFoundSingleInputOnly() {
try {
SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
SourcePlanNode otherTarget = new SourcePlanNode(getSourceNode(), "Source");
Channel toMap1 = new Channel(target);
toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap1.setLocalStrategy(LocalStrategy.NONE);
SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
Channel toMap2 = new Channel(map1);
toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
toMap2.setLocalStrategy(LocalStrategy.NONE);
SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
{
GlobalProperties gp = new GlobalProperties();
LocalProperties lp = new LocalProperties();
FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(otherTarget, gp, lp);
assertTrue(report == NO_PARTIAL_SOLUTION);
}
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
