org.apache.flink.runtime.operators.testutils.UniformRecordGenerator Java Examples
The following examples show how to use
org.apache.flink.runtime.operators.testutils.UniformRecordGenerator.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: ReduceTaskTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Test
public void testFailingReduceTask() {
final int keyCnt = 100;
final int valCnt = 20;
addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
addDriverComparator(this.comparator);
setOutput(this.outList);
getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
try {
testDriver(testTask, MockFailingReduceStub.class);
Assert.fail("Function exception was not forwarded.");
} catch (ExpectedTestException eetex) {
// Good!
} catch (Exception e) {
LOG.info("Exception which was not the ExpectedTestException while running the test task.", e);
Assert.fail("Test caused exception: " + e.getMessage());
}
this.outList.clear();
}
Example #2
| Source File: ReduceTaskTest.java From flink with Apache License 2.0 | 6 votes |
|
@Test
public void testFailingReduceTask() {
final int keyCnt = 100;
final int valCnt = 20;
addInput(new UniformRecordGenerator(keyCnt, valCnt, true));
addDriverComparator(this.comparator);
setOutput(this.outList);
getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
try {
testDriver(testTask, MockFailingReduceStub.class);
Assert.fail("Function exception was not forwarded.");
} catch (ExpectedTestException eetex) {
// Good!
} catch (Exception e) {
LOG.info("Exception which was not the ExpectedTestException while running the test task.", e);
Assert.fail("Test caused exception: " + e.getMessage());
}
this.outList.clear();
}
Example #3
| Source File: FlatMapTaskTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Test
public void testMapTask() {
final int keyCnt = 100;
final int valCnt = 20;
addInput(new UniformRecordGenerator(keyCnt, valCnt, false));
setOutput(this.output);
final FlatMapDriver<Record, Record> testDriver = new FlatMapDriver<>();
try {
testDriver(testDriver, MockMapStub.class);
} catch (Exception e) {
LOG.debug("Exception while running the test driver.", e);
Assert.fail("Invoke method caused exception.");
}
Assert.assertEquals("Wrong result set size.", keyCnt*valCnt, this.output.getNumberOfRecords());
}
Example #4
| Source File: JoinTaskExternalITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testExternalSort1MatchTask() {
final int keyCnt1 = 16384*4;
final int valCnt1 = 2;
final int keyCnt2 = 8192;
final int valCnt2 = 4*2;
final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
setOutput(this.output);
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
setNumFileHandlesForSort(4);
final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
try {
addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
testDriver(testTask, MockMatchStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("The test caused an exception.");
}
Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #5
| Source File: ReduceTaskTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testReduceTaskWithSortingInput() {
final int keyCnt = 100;
final int valCnt = 20;
addDriverComparator(this.comparator);
setOutput(this.outList);
getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
try {
addInputSorted(new UniformRecordGenerator(keyCnt, valCnt, false), this.comparator.duplicate());
GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
testDriver(testTask, MockReduceStub.class);
} catch (Exception e) {
LOG.info("Exception while running the test task.", e);
Assert.fail("Exception in Test: " + e.getMessage());
}
Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+keyCnt, this.outList.size() == keyCnt);
for(Record record : this.outList) {
Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == valCnt-record.getField(0, IntValue.class).getValue());
}
this.outList.clear();
}
Example #6
| Source File: JoinTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSortBoth1MatchTask() {
final int keyCnt1 = 20;
final int valCnt1 = 1;
final int keyCnt2 = 10;
final int valCnt2 = 2;
setOutput(this.outList);
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
setNumFileHandlesForSort(4);
final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
try {
addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
testDriver(testTask, MockMatchStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("The test caused an exception.");
}
final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
Assert.assertTrue("Resultset size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);
this.outList.clear();
}
Example #7
| Source File: CoGroupTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSortSecondCoGroupTask() {
int keyCnt1 = 200;
int valCnt1 = 2;
int keyCnt2 = 200;
int valCnt2 = 4;
final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2) +
(keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
setOutput(this.output);
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
try {
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
testDriver(testTask, MockCoGroupStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("The test caused an exception.");
}
Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #8
| Source File: CoGroupTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testMergeCoGroupTask() {
int keyCnt1 = 200;
int valCnt1 = 2;
int keyCnt2 = 200;
int valCnt2 = 4;
final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2) +
(keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
setOutput(this.output);
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, true));
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
try {
testDriver(testTask, MockCoGroupStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("The test caused an exception.");
}
Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #9
| Source File: CrossTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testFailingBlockCrossTask2() {
int keyCnt1 = 10;
int valCnt1 = 1;
int keyCnt2 = 100;
int valCnt2 = 4;
setOutput(this.output);
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_BLOCKED_OUTER_SECOND);
getTaskConfig().setRelativeMemoryDriver(cross_frac);
final CrossDriver<Record, Record, Record> testTask = new CrossDriver<>();
try {
testDriver(testTask, MockFailingCrossStub.class);
Assert.fail("Exception not forwarded.");
} catch (ExpectedTestException etex) {
// good!
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test failed due to an exception.");
}
}
Example #10
| Source File: CachedMatchTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testHash2MatchTask() {
int keyCnt1 = 20;
int valCnt1 = 1;
int keyCnt2 = 20;
int valCnt2 = 1;
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
setOutput(this.outList);
getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED);
getTaskConfig().setRelativeMemoryDriver(1.0f);
BuildSecondCachedJoinDriver<Record, Record, Record> testTask = new BuildSecondCachedJoinDriver<Record, Record, Record>();
try {
testResettableDriver(testTask, MockMatchStub.class, 3);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test caused an exception.");
}
final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
this.outList.clear();
}
Example #11
| Source File: CachedMatchTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testHash3MatchTask() {
int keyCnt1 = 20;
int valCnt1 = 1;
int keyCnt2 = 20;
int valCnt2 = 20;
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
setOutput(this.outList);
getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
getTaskConfig().setRelativeMemoryDriver(1.0f);
BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
try {
testResettableDriver(testTask, MockMatchStub.class, 3);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test caused an exception.");
}
final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
this.outList.clear();
}
Example #12
| Source File: CachedMatchTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testHash4MatchTask() {
int keyCnt1 = 20;
int valCnt1 = 20;
int keyCnt2 = 20;
int valCnt2 = 1;
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
setOutput(this.outList);
getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED);
getTaskConfig().setRelativeMemoryDriver(1.0f);
BuildSecondCachedJoinDriver<Record, Record, Record> testTask = new BuildSecondCachedJoinDriver<Record, Record, Record>();
try {
testResettableDriver(testTask, MockMatchStub.class, 3);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test caused an exception.");
}
final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
this.outList.clear();
}
Example #13
| Source File: CachedMatchTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testHash5MatchTask() {
int keyCnt1 = 20;
int valCnt1 = 20;
int keyCnt2 = 20;
int valCnt2 = 20;
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
setOutput(this.outList);
getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
getTaskConfig().setRelativeMemoryDriver(1.0f);
BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
try {
testResettableDriver(testTask, MockMatchStub.class, 3);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test caused an exception.");
}
final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
this.outList.clear();
}
Example #14
| Source File: CachedMatchTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testFailingHashFirstMatchTask() {
int keyCnt1 = 20;
int valCnt1 = 20;
int keyCnt2 = 20;
int valCnt2 = 20;
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
setOutput(new NirvanaOutputList());
getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
getTaskConfig().setRelativeMemoryDriver(1.0f);
BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
try {
testResettableDriver(testTask, MockFailingMatchStub.class, 3);
Assert.fail("Function exception was not forwarded.");
} catch (ExpectedTestException etex) {
// good!
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test caused an exception.");
}
}
Example #15
| Source File: CoGroupTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSortFirstCoGroupTask() {
int keyCnt1 = 200;
int valCnt1 = 2;
int keyCnt2 = 200;
int valCnt2 = 4;
final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2) +
(keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
setOutput(this.output);
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
try {
addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, true));
testDriver(testTask, MockCoGroupStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("The test caused an exception.");
}
Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #16
| Source File: JoinTaskExternalITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testExternalHash1MatchTask() {
final int keyCnt1 = 32768;
final int valCnt1 = 8;
final int keyCnt2 = 65536;
final int valCnt2 = 8;
final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
setOutput(this.output);
getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
getTaskConfig().setRelativeMemoryDriver(hash_frac);
JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
try {
testDriver(testTask, MockMatchStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test caused an exception.");
}
Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #17
| Source File: JoinTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSortSecondMatchTask() {
int keyCnt1 = 20;
int valCnt1 = 20;
int keyCnt2 = 20;
int valCnt2 = 20;
setOutput(this.outList);
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
setNumFileHandlesForSort(4);
final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
try {
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
testDriver(testTask, MockMatchStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("The test caused an exception.");
}
int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
this.outList.clear();
}
Example #18
| Source File: ReduceTaskExternalITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testMultiLevelMergeReduceTask() {
final int keyCnt = 32768;
final int valCnt = 8;
setNumFileHandlesForSort(2);
addDriverComparator(this.comparator);
setOutput(this.outList);
getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
try {
addInputSorted(new UniformRecordGenerator(keyCnt, valCnt, false), this.comparator.duplicate());
GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
testDriver(testTask, MockReduceStub.class);
} catch (Exception e) {
LOG.info("Exception while running the test task.", e);
Assert.fail("Exception in Test: " + e.getMessage());
}
Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+keyCnt, this.outList.size() == keyCnt);
for(Record record : this.outList) {
Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == valCnt-record.getField(0, IntValue.class).getValue());
}
this.outList.clear();
}
Example #19
| Source File: ReduceTaskExternalITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testSingleLevelMergeReduceTask() {
final int keyCnt = 8192;
final int valCnt = 8;
setNumFileHandlesForSort(2);
addDriverComparator(this.comparator);
setOutput(this.outList);
getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
try {
addInputSorted(new UniformRecordGenerator(keyCnt, valCnt, false), this.comparator.duplicate());
GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
testDriver(testTask, MockReduceStub.class);
} catch (Exception e) {
LOG.info("Exception while running the test task.", e);
Assert.fail("Exception in Test: " + e.getMessage());
}
Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+keyCnt, this.outList.size() == keyCnt);
for(Record record : this.outList) {
Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == valCnt-record.getField(0, IntValue.class).getValue());
}
this.outList.clear();
}
Example #20
| Source File: CoGroupTaskExternalITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testExternalSortCoGroupTask() {
int keyCnt1 = 16384*8;
int valCnt1 = 32;
int keyCnt2 = 65536*4;
int valCnt2 = 4;
final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2) +
(keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
setOutput(this.output);
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
try {
addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
testDriver(testTask, MockCoGroupStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("The test caused an exception.");
}
Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #21
| Source File: CrossTaskExternalITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testExternalStreamCrossTask() {
int keyCnt1 = 2;
int valCnt1 = 1;
// 87381 fit into memory, 87382 do not!
int keyCnt2 = 87385;
int valCnt2 = 1;
final int expCnt = keyCnt1*valCnt1*keyCnt2*valCnt2;
setOutput(this.output);
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_STREAMED_OUTER_FIRST);
getTaskConfig().setRelativeMemoryDriver(cross_frac);
final CrossDriver<Record, Record, Record> testTask = new CrossDriver<Record, Record, Record>();
try {
testDriver(testTask, MockCrossStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test failed due to an exception.");
}
Assert.assertEquals("Wrong result size.", expCnt, this.output.getNumberOfRecords());
}
Example #22
| Source File: CrossTaskExternalITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testExternalBlockCrossTask() {
int keyCnt1 = 2;
int valCnt1 = 1;
// 43690 fit into memory, 43691 do not!
int keyCnt2 = 43700;
int valCnt2 = 1;
final int expCnt = keyCnt1*valCnt1*keyCnt2*valCnt2;
setOutput(this.output);
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_BLOCKED_OUTER_FIRST);
getTaskConfig().setRelativeMemoryDriver(cross_frac);
final CrossDriver<Record, Record, Record> testTask = new CrossDriver<Record, Record, Record>();
try {
testDriver(testTask, MockCrossStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test failed due to an exception.");
}
Assert.assertEquals("Wrong result size.", expCnt, this.output.getNumberOfRecords());
}
Example #23
| Source File: CrossTaskTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testBlockEmptyOuterCrossTask() {
int keyCnt1 = 10;
int valCnt1 = 1;
int keyCnt2 = 0;
int valCnt2 = 0;
final int expCnt = keyCnt1*valCnt1*keyCnt2*valCnt2;
setOutput(this.output);
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_BLOCKED_OUTER_SECOND);
getTaskConfig().setRelativeMemoryDriver(cross_frac);
final CrossDriver<Record, Record, Record> testTask = new CrossDriver<>();
try {
testDriver(testTask, MockCrossStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test failed due to an exception.");
}
Assert.assertEquals("Wrong result size.", expCnt, this.output.getNumberOfRecords());
}
Example #24
| Source File: JoinTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testSortBoth3MatchTask() {
int keyCnt1 = 20;
int valCnt1 = 1;
int keyCnt2 = 20;
int valCnt2 = 20;
setOutput(this.outList);
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
setNumFileHandlesForSort(4);
final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
try {
addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
testDriver(testTask, MockMatchStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("The test caused an exception.");
}
int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
this.outList.clear();
}
Example #25
| Source File: CrossTaskTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testStreamEmptyOuterCrossTask() {
int keyCnt1 = 10;
int valCnt1 = 1;
int keyCnt2 = 0;
int valCnt2 = 0;
final int expCnt = keyCnt1*valCnt1*keyCnt2*valCnt2;
setOutput(this.output);
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_STREAMED_OUTER_SECOND);
getTaskConfig().setRelativeMemoryDriver(cross_frac);
final CrossDriver<Record, Record, Record> testTask = new CrossDriver<>();
try {
testDriver(testTask, MockCrossStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test failed due to an exception.");
}
Assert.assertEquals("Wrong result size.", expCnt, this.output.getNumberOfRecords());
}
Example #26
| Source File: CoGroupTaskTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testSortBoth1CoGroupTask() {
int keyCnt1 = 100;
int valCnt1 = 2;
int keyCnt2 = 200;
int valCnt2 = 1;
final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2) +
(keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
setOutput(this.output);
addDriverComparator(this.comparator1);
addDriverComparator(this.comparator2);
getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
try {
addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
testDriver(testTask, MockCoGroupStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("The test caused an exception.");
}
Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #27
| Source File: CrossTaskTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testFailingStreamCrossTask2() {
int keyCnt1 = 10;
int valCnt1 = 1;
int keyCnt2 = 100;
int valCnt2 = 4;
setOutput(this.output);
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_STREAMED_OUTER_SECOND);
getTaskConfig().setRelativeMemoryDriver(cross_frac);
final CrossDriver<Record, Record, Record> testTask = new CrossDriver<>();
try {
testDriver(testTask, MockFailingCrossStub.class);
Assert.fail("Exception not forwarded.");
} catch (ExpectedTestException etex) {
// good!
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test failed due to an exception.");
}
}
Example #28
| Source File: CrossTaskTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testFailingStreamCrossTask() {
int keyCnt1 = 10;
int valCnt1 = 1;
int keyCnt2 = 100;
int valCnt2 = 4;
setOutput(this.output);
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_STREAMED_OUTER_FIRST);
getTaskConfig().setRelativeMemoryDriver(cross_frac);
final CrossDriver<Record, Record, Record> testTask = new CrossDriver<>();
try {
testDriver(testTask, MockFailingCrossStub.class);
Assert.fail("Exception not forwarded.");
} catch (ExpectedTestException etex) {
// good!
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test failed due to an exception.");
}
}
Example #29
| Source File: CrossTaskTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testStream2CrossTask() {
int keyCnt1 = 10;
int valCnt1 = 1;
int keyCnt2 = 100;
int valCnt2 = 4;
final int expCnt = keyCnt1*valCnt1*keyCnt2*valCnt2;
setOutput(this.output);
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_STREAMED_OUTER_SECOND);
getTaskConfig().setRelativeMemoryDriver(cross_frac);
final CrossDriver<Record, Record, Record> testTask = new CrossDriver<>();
try {
testDriver(testTask, MockCrossStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test failed due to an exception.");
}
Assert.assertEquals("Wrong result size.", expCnt, this.output.getNumberOfRecords());
}
Example #30
| Source File: CrossTaskTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testStream1CrossTask() {
int keyCnt1 = 10;
int valCnt1 = 1;
int keyCnt2 = 100;
int valCnt2 = 4;
final int expCnt = keyCnt1*valCnt1*keyCnt2*valCnt2;
setOutput(this.output);
addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_STREAMED_OUTER_FIRST);
getTaskConfig().setRelativeMemoryDriver(cross_frac);
final CrossDriver<Record, Record, Record> testTask = new CrossDriver<>();
try {
testDriver(testTask, MockCrossStub.class);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Test failed due to an exception.");
}
Assert.assertEquals("Wrong result size.", expCnt, this.output.getNumberOfRecords());
}
