org.apache.flink.api.common.operators.SemanticProperties Java Examples
The following examples show how to use
org.apache.flink.api.common.operators.SemanticProperties.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: GroupReduceNode.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Override
protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() {
// Local properties for GroupReduce may only be preserved on key fields.
SingleInputSemanticProperties origProps = getOperator().getSemanticProperties();
SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties();
FieldSet readSet = origProps.getReadFields(0);
if(readSet != null) {
filteredProps.addReadFields(readSet);
}
// only add forward field information for key fields
if(this.keys != null) {
for (int f : this.keys) {
FieldSet targets = origProps.getForwardingTargetFields(0, f);
for (int t : targets) {
filteredProps.addForwardedField(f, t);
}
}
}
return filteredProps;
}
Example #2
| Source File: GroupReduceNode.java From flink with Apache License 2.0 | 6 votes |
|
@Override
protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() {
// Local properties for GroupReduce may only be preserved on key fields.
SingleInputSemanticProperties origProps = getOperator().getSemanticProperties();
SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties();
FieldSet readSet = origProps.getReadFields(0);
if(readSet != null) {
filteredProps.addReadFields(readSet);
}
// only add forward field information for key fields
if(this.keys != null) {
for (int f : this.keys) {
FieldSet targets = origProps.getForwardingTargetFields(0, f);
for (int t : targets) {
filteredProps.addForwardedField(f, t);
}
}
}
return filteredProps;
}
Example #3
| Source File: GroupCombineNode.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Override
protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() {
// Local properties for GroupCombine may only be preserved on key fields.
SingleInputSemanticProperties origProps =
((SingleInputOperator<?,?,?>) getOperator()).getSemanticProperties();
SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties();
FieldSet readSet = origProps.getReadFields(0);
if(readSet != null) {
filteredProps.addReadFields(readSet);
}
// only add forward field information for key fields
if(this.keys != null) {
for (int f : this.keys) {
FieldSet targets = origProps.getForwardingTargetFields(0, f);
for (int t : targets) {
filteredProps.addForwardedField(f, t);
}
}
}
return filteredProps;
}
Example #4
| Source File: ReduceOperatorTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSemanticPropsWithKeySelector3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
ReduceOperator<Tuple5<Integer, Long, String, Long, Integer>> reduceOp =
tupleDs.groupBy(new DummyTestKeySelector())
.reduce(new DummyReduceFunction3())
.withForwardedFields("4->0;3;3->1;2");
SemanticProperties semProps = reduceOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 2);
assertTrue(semProps.getForwardingTargetFields(0, 5).contains(1));
assertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 6).contains(0));
assertTrue(semProps.getForwardingSourceField(0, 0) == 6);
assertTrue(semProps.getForwardingSourceField(0, 1) == 5);
assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
assertTrue(semProps.getForwardingSourceField(0, 3) == 5);
assertTrue(semProps.getForwardingSourceField(0, 4) < 0);
assertTrue(semProps.getReadFields(0) == null);
}
Example #5
| Source File: MapPartitionNode.java From flink with Apache License 2.0 | 5 votes |
|
@Override
protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() {
// Local properties for MapPartition may not be preserved.
SingleInputSemanticProperties origProps =
((SingleInputOperator<?,?,?>) getOperator()).getSemanticProperties();
SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties();
FieldSet readSet = origProps.getReadFields(0);
if(readSet != null) {
filteredProps.addReadFields(readSet);
}
return filteredProps;
}
Example #6
| Source File: GroupReduceNodeTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testGetSemanticProperties() {
SingleInputSemanticProperties origProps = new SingleInputSemanticProperties();
origProps.addForwardedField(0, 1);
origProps.addForwardedField(2, 2);
origProps.addForwardedField(3, 4);
origProps.addForwardedField(6, 0);
origProps.addReadFields(new FieldSet(0, 2, 4, 7));
GroupReduceOperatorBase<?,?,?> op = mock(GroupReduceOperatorBase.class);
when(op.getSemanticProperties()).thenReturn(origProps);
when(op.getKeyColumns(0)).thenReturn(new int[]{3,2});
when(op.getParameters()).thenReturn(new Configuration());
GroupReduceNode node = new GroupReduceNode(op);
SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();
assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);
assertTrue(filteredProps.getReadFields(0).size() == 4);
assertTrue(filteredProps.getReadFields(0).contains(0));
assertTrue(filteredProps.getReadFields(0).contains(2));
assertTrue(filteredProps.getReadFields(0).contains(4));
assertTrue(filteredProps.getReadFields(0).contains(7));
}
Example #7
| Source File: GroupCombineOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testSemanticPropsWithKeySelector5() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupCombineOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> combineOp =
tupleDs.groupBy(new DummyTestKeySelector())
.combineGroup(new DummyGroupCombineFunction3())
.withForwardedFields("4->0;3;3->1;2");
SemanticProperties semProps = combineOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 2);
assertTrue(semProps.getForwardingTargetFields(0, 5).contains(1));
assertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 6).contains(0));
assertTrue(semProps.getForwardingSourceField(0, 0) == 6);
assertTrue(semProps.getForwardingSourceField(0, 1) == 5);
assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
assertTrue(semProps.getForwardingSourceField(0, 3) == 5);
assertTrue(semProps.getForwardingSourceField(0, 4) < 0);
assertTrue(semProps.getReadFields(0) == null);
}
Example #8
| Source File: GroupCombineOperatorTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSemanticPropsWithKeySelector3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupCombineOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> combineOp =
tupleDs.groupBy(new DummyTestKeySelector())
.combineGroup(new DummyGroupCombineFunction2())
.withForwardedFields("0->4;1;1->3;2");
SemanticProperties semProps = combineOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
assertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
assertTrue(semProps.getForwardingSourceField(0, 3) == 3);
assertTrue(semProps.getForwardingSourceField(0, 4) == 2);
assertTrue(semProps.getReadFields(0).size() == 3);
assertTrue(semProps.getReadFields(0).contains(2));
assertTrue(semProps.getReadFields(0).contains(5));
assertTrue(semProps.getReadFields(0).contains(6));
}
Example #9
| Source File: SemanticPropertiesTranslationTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class)
public void testBinaryForwardedOverwritingInLine5() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings("unchecked")
DataSet<Tuple3<Long, Long, Long>> input1 = env.fromElements(new Tuple3<Long, Long, Long>(3L, 4L, 5L));
@SuppressWarnings("unchecked")
DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(3L, 2L, 1L));
input1.join(input2).where(0).equalTo(0).with(new AllForwardedExceptJoin<Long>())
.withForwardedFieldsFirst("0->1;");
}
Example #10
| Source File: GroupReduceOperatorTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSemanticPropsWithKeySelector7() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupReduceOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> reduceOp =
tupleDs.groupBy(new DummyTestKeySelector())
.reduceGroup(new DummyGroupReduceFunction4());
SemanticProperties semProps = reduceOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 2).contains(0));
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
assertTrue(semProps.getForwardingSourceField(0, 0) == 2);
assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
assertTrue(semProps.getForwardingSourceField(0, 2) < 0);
assertTrue(semProps.getForwardingSourceField(0, 3) == 5);
assertTrue(semProps.getForwardingSourceField(0, 4) < 0);
assertTrue(semProps.getReadFields(0) == null);
}
Example #11
| Source File: GroupCombineOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testSemanticPropsWithKeySelector3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupCombineOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> combineOp =
tupleDs.groupBy(new DummyTestKeySelector())
.combineGroup(new DummyGroupCombineFunction2())
.withForwardedFields("0->4;1;1->3;2");
SemanticProperties semProps = combineOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
assertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
assertTrue(semProps.getForwardingSourceField(0, 3) == 3);
assertTrue(semProps.getForwardingSourceField(0, 4) == 2);
assertTrue(semProps.getReadFields(0).size() == 3);
assertTrue(semProps.getReadFields(0).contains(2));
assertTrue(semProps.getReadFields(0).contains(5));
assertTrue(semProps.getReadFields(0).contains(6));
}
Example #12
| Source File: GroupCombineOperatorTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSemanticPropsWithKeySelector4() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupCombineOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> combineOp =
tupleDs.groupBy(new DummyTestKeySelector())
.sortGroup(new DummyTestKeySelector(), Order.ASCENDING)
.combineGroup(new DummyGroupCombineFunction2())
.withForwardedFields("0->4;1;1->3;2");
SemanticProperties semProps = combineOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 4).contains(4));
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 2);
assertTrue(semProps.getForwardingTargetFields(0, 5).contains(1));
assertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 6).contains(2));
assertTrue(semProps.getForwardingTargetFields(0, 7).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 8).size() == 0);
assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
assertTrue(semProps.getForwardingSourceField(0, 1) == 5);
assertTrue(semProps.getForwardingSourceField(0, 2) == 6);
assertTrue(semProps.getForwardingSourceField(0, 3) == 5);
assertTrue(semProps.getForwardingSourceField(0, 4) == 4);
assertTrue(semProps.getReadFields(0).size() == 3);
assertTrue(semProps.getReadFields(0).contains(4));
assertTrue(semProps.getReadFields(0).contains(7));
assertTrue(semProps.getReadFields(0).contains(8));
}
Example #13
| Source File: GroupCombineOperatorTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSemanticPropsWithKeySelector2() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupCombineOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> combineOp =
tupleDs.groupBy(new DummyTestKeySelector())
.sortGroup(new DummyTestKeySelector(), Order.ASCENDING)
.combineGroup(new DummyGroupCombineFunction1());
SemanticProperties semProps = combineOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 4).contains(4));
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 2);
assertTrue(semProps.getForwardingTargetFields(0, 5).contains(1));
assertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 6).contains(2));
assertTrue(semProps.getForwardingTargetFields(0, 7).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 8).size() == 0);
assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
assertTrue(semProps.getForwardingSourceField(0, 1) == 5);
assertTrue(semProps.getForwardingSourceField(0, 2) == 6);
assertTrue(semProps.getForwardingSourceField(0, 3) == 5);
assertTrue(semProps.getForwardingSourceField(0, 4) == 4);
assertTrue(semProps.getReadFields(0).size() == 3);
assertTrue(semProps.getReadFields(0).contains(4));
assertTrue(semProps.getReadFields(0).contains(7));
assertTrue(semProps.getReadFields(0).contains(8));
}
Example #14
| Source File: JoinOperatorTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSemanticPropsWithKeySelector3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs2 = env.fromCollection(emptyTupleData, tupleTypeInfo);
JoinOperator<?, ?, ? extends Tuple> joinOp = tupleDs1.join(tupleDs2)
.where(new DummyTestKeySelector()).equalTo(new DummyTestKeySelector())
.projectFirst(2)
.projectSecond(0, 0, 3)
.projectFirst(0, 4)
.projectSecond(2);
SemanticProperties semProps = joinOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 4).contains(0));
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 6).contains(5));
assertTrue(semProps.getForwardingTargetFields(1, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(1, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(1, 2).size() == 2);
assertTrue(semProps.getForwardingTargetFields(1, 2).contains(1));
assertTrue(semProps.getForwardingTargetFields(1, 2).contains(2));
assertTrue(semProps.getForwardingTargetFields(1, 3).size() == 0);
assertTrue(semProps.getForwardingTargetFields(1, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(1, 4).contains(6));
assertTrue(semProps.getForwardingTargetFields(1, 5).size() == 1);
assertTrue(semProps.getForwardingTargetFields(1, 5).contains(3));
assertTrue(semProps.getForwardingTargetFields(1, 6).size() == 0);
}
Example #15
| Source File: GroupCombineOperatorTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSemanticPropsWithKeySelector1() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupCombineOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> combineOp =
tupleDs.groupBy(new DummyTestKeySelector())
.combineGroup(new DummyGroupCombineFunction1());
SemanticProperties semProps = combineOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
assertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
assertTrue(semProps.getForwardingSourceField(0, 3) == 3);
assertTrue(semProps.getForwardingSourceField(0, 4) == 2);
assertTrue(semProps.getReadFields(0).size() == 3);
assertTrue(semProps.getReadFields(0).contains(2));
assertTrue(semProps.getReadFields(0).contains(5));
assertTrue(semProps.getReadFields(0).contains(6));
}
Example #16
| Source File: SemanticPropertiesTranslationTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class)
public void testBinaryForwardedOverwritingInLine6() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings("unchecked")
DataSet<Tuple3<Long, Long, Long>> input1 = env.fromElements(new Tuple3<Long, Long, Long>(3L, 4L, 5L));
@SuppressWarnings("unchecked")
DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(3L, 2L, 1L));
input1.join(input2).where(0).equalTo(0).with(new AllForwardedExceptJoin<Long>())
.withForwardedFieldsSecond("0->1;");
}
Example #17
| Source File: GroupReduceOperatorTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSemanticPropsWithKeySelector3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupReduceOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> reduceOp =
tupleDs.groupBy(new DummyTestKeySelector())
.reduceGroup(new DummyGroupReduceFunction2())
.withForwardedFields("0->4;1;1->3;2");
SemanticProperties semProps = reduceOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
assertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
assertTrue(semProps.getForwardingSourceField(0, 3) == 3);
assertTrue(semProps.getForwardingSourceField(0, 4) == 2);
assertTrue(semProps.getReadFields(0).size() == 3);
assertTrue(semProps.getReadFields(0).contains(2));
assertTrue(semProps.getReadFields(0).contains(5));
assertTrue(semProps.getReadFields(0).contains(6));
}
Example #18
| Source File: SemanticPropertiesTranslationTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class)
public void testUnaryForwardedOverwritingInLine1() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings("unchecked")
DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(3L, 2L, 1L));
input.map(new WildcardForwardedMapper<Tuple3<Long, Long, Long>>()).withForwardedFields("0->1; 2");
}
Example #19
| Source File: SemanticPropertiesTranslationTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class)
public void testBinaryForwardedOverwritingInLine1() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings("unchecked")
DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L));
@SuppressWarnings("unchecked")
DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L));
input1.join(input2).where(0).equalTo(0).with(new ForwardedFirstAnnotationJoin<Long>())
.withForwardedFieldsFirst("0->1");
}
Example #20
| Source File: SemanticPropertiesTranslationTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class)
public void testBinaryForwardedOverwritingInLine2() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings("unchecked")
DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L));
@SuppressWarnings("unchecked")
DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L));
input1.join(input2).where(0).equalTo(0).with(new ForwardedSecondAnnotationJoin<Long>())
.withForwardedFieldsSecond("0->1");
}
Example #21
| Source File: GroupReduceOperatorTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSemanticPropsWithKeySelector4() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupReduceOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> reduceOp =
tupleDs.groupBy(new DummyTestKeySelector())
.sortGroup(new DummyTestKeySelector(), Order.ASCENDING)
.reduceGroup(new DummyGroupReduceFunction2())
.withForwardedFields("0->4;1;1->3;2");
SemanticProperties semProps = reduceOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 4).contains(4));
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 2);
assertTrue(semProps.getForwardingTargetFields(0, 5).contains(1));
assertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 6).contains(2));
assertTrue(semProps.getForwardingTargetFields(0, 7).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 8).size() == 0);
assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
assertTrue(semProps.getForwardingSourceField(0, 1) == 5);
assertTrue(semProps.getForwardingSourceField(0, 2) == 6);
assertTrue(semProps.getForwardingSourceField(0, 3) == 5);
assertTrue(semProps.getForwardingSourceField(0, 4) == 4);
assertTrue(semProps.getReadFields(0).size() == 3);
assertTrue(semProps.getReadFields(0).contains(4));
assertTrue(semProps.getReadFields(0).contains(7));
assertTrue(semProps.getReadFields(0).contains(8));
}
Example #22
| Source File: SemanticPropertiesTranslationTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class)
public void testBinaryForwardedOverwritingInLine4() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings("unchecked")
DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L));
@SuppressWarnings("unchecked")
DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L));
input1.join(input2).where(0).equalTo(0).with(new ForwardedBothAnnotationJoin<Long, Long, Long, Long>())
.withForwardedFieldsSecond("0->1;");
}
Example #23
| Source File: SemanticPropertiesTranslationTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class)
public void testBinaryForwardedOverwritingInLine5() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings("unchecked")
DataSet<Tuple3<Long, Long, Long>> input1 = env.fromElements(new Tuple3<Long, Long, Long>(3L, 4L, 5L));
@SuppressWarnings("unchecked")
DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(3L, 2L, 1L));
input1.join(input2).where(0).equalTo(0).with(new AllForwardedExceptJoin<Long>())
.withForwardedFieldsFirst("0->1;");
}
Example #24
| Source File: SemanticPropertiesTranslationTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class)
public void testBinaryForwardedOverwritingInLine6() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings("unchecked")
DataSet<Tuple3<Long, Long, Long>> input1 = env.fromElements(new Tuple3<Long, Long, Long>(3L, 4L, 5L));
@SuppressWarnings("unchecked")
DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(3L, 2L, 1L));
input1.join(input2).where(0).equalTo(0).with(new AllForwardedExceptJoin<Long>())
.withForwardedFieldsSecond("0->1;");
}
Example #25
| Source File: GroupCombineNodeTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testGetSemanticProperties() {
SingleInputSemanticProperties origProps = new SingleInputSemanticProperties();
origProps.addForwardedField(0, 1);
origProps.addForwardedField(2, 2);
origProps.addForwardedField(3, 4);
origProps.addForwardedField(6, 0);
origProps.addReadFields(new FieldSet(0, 2, 4, 7));
GroupCombineOperatorBase<?,?,?> op = mock(GroupCombineOperatorBase.class);
when(op.getSemanticProperties()).thenReturn(origProps);
when(op.getKeyColumns(0)).thenReturn(new int[]{3,2});
when(op.getParameters()).thenReturn(new Configuration());
GroupCombineNode node = new GroupCombineNode(op);
SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();
assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);
assertTrue(filteredProps.getReadFields(0).size() == 4);
assertTrue(filteredProps.getReadFields(0).contains(0));
assertTrue(filteredProps.getReadFields(0).contains(2));
assertTrue(filteredProps.getReadFields(0).contains(4));
assertTrue(filteredProps.getReadFields(0).contains(7));
}
Example #26
| Source File: MapPartitionNodeTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testGetSemanticProperties() {
SingleInputSemanticProperties origProps = new SingleInputSemanticProperties();
origProps.addForwardedField(0, 1);
origProps.addForwardedField(2, 2);
origProps.addReadFields(new FieldSet(0, 2, 4, 7));
MapPartitionOperatorBase<?,?,?> op = mock(MapPartitionOperatorBase.class);
when(op.getSemanticProperties()).thenReturn(origProps);
when(op.getKeyColumns(0)).thenReturn(new int[]{});
MapPartitionNode node = new MapPartitionNode(op);
SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();
assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 0);
assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
assertTrue(filteredProps.getForwardingSourceField(0, 2) < 0);
assertTrue(filteredProps.getReadFields(0).size() == 4);
assertTrue(filteredProps.getReadFields(0).contains(0));
assertTrue(filteredProps.getReadFields(0).contains(2));
assertTrue(filteredProps.getReadFields(0).contains(4));
assertTrue(filteredProps.getReadFields(0).contains(7));
}
Example #27
| Source File: GroupReduceOperatorTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSemanticPropsWithKeySelector1() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupReduceOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> reduceOp =
tupleDs.groupBy(new DummyTestKeySelector())
.reduceGroup(new DummyGroupReduceFunction1());
SemanticProperties semProps = reduceOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
assertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
assertTrue(semProps.getForwardingSourceField(0, 3) == 3);
assertTrue(semProps.getForwardingSourceField(0, 4) == 2);
assertTrue(semProps.getReadFields(0).size() == 3);
assertTrue(semProps.getReadFields(0).contains(2));
assertTrue(semProps.getReadFields(0).contains(5));
assertTrue(semProps.getReadFields(0).contains(6));
}
Example #28
| Source File: GroupReduceOperatorTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSemanticPropsWithKeySelector5() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupReduceOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> reduceOp =
tupleDs.groupBy(new DummyTestKeySelector())
.reduceGroup(new DummyGroupReduceFunction3())
.withForwardedFields("4->0;3;3->1;2");
SemanticProperties semProps = reduceOp.getSemanticProperties();
assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 2);
assertTrue(semProps.getForwardingTargetFields(0, 5).contains(1));
assertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
assertTrue(semProps.getForwardingTargetFields(0, 6).contains(0));
assertTrue(semProps.getForwardingSourceField(0, 0) == 6);
assertTrue(semProps.getForwardingSourceField(0, 1) == 5);
assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
assertTrue(semProps.getForwardingSourceField(0, 3) == 5);
assertTrue(semProps.getForwardingSourceField(0, 4) < 0);
assertTrue(semProps.getReadFields(0) == null);
}
Example #29
| Source File: SemanticPropertiesTranslationTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class)
public void testBinaryForwardedOverwritingInLine4() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings("unchecked")
DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L));
@SuppressWarnings("unchecked")
DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L));
input1.join(input2).where(0).equalTo(0).with(new ForwardedBothAnnotationJoin<Long, Long, Long, Long>())
.withForwardedFieldsSecond("0->1;");
}
Example #30
| Source File: SemanticPropertiesTranslationTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class)
public void testUnaryForwardedOverwritingInLine2() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings("unchecked")
DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(3L, 2L, 1L));
input.map(new AllForwardedExceptMapper<Tuple3<Long, Long, Long>>()).withForwardedFields("0->1; 2");
}
