org.apache.flink.runtime.testutils.recordutils.RecordPairComparatorFactory Java Examples

The following examples show how to use org.apache.flink.runtime.testutils.recordutils.RecordPairComparatorFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testHash1MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 10;
	int valCnt2 = 2;
			
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
Example #2
Source File: CachedMatchTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testHash5MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockMatchStub.class, 3);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
Example #3
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testFailingHashFirstMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(new NirvanaOutputList());
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockFailingMatchStub.class);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException etex) {
		// good!
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
}
Example #4
Source File: CachedMatchTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testFailingHashSecondMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(new NirvanaOutputList());
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildSecondCachedJoinDriver<Record, Record, Record> testTask = new BuildSecondCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockFailingMatchStub.class, 3);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException etex) {
		// good!
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
}
Example #5
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testMergeMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, true));
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
Example #6
Source File: CachedMatchTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testFailingHashFirstMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(new NirvanaOutputList());
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockFailingMatchStub.class, 3);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException etex) {
		// good!
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
}
Example #7
Source File: CachedMatchTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testHash3MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockMatchStub.class, 3);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
Example #8
Source File: CachedMatchTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testHash2MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 20;
	int valCnt2 = 1;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildSecondCachedJoinDriver<Record, Record, Record> testTask = new BuildSecondCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockMatchStub.class, 3);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
Example #9
Source File: JoinTaskExternalITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testExternalSort1MatchTask() {
	final int keyCnt1 = 16384*4;
	final int valCnt1 = 2;
	
	final int keyCnt2 = 8192;
	final int valCnt2 = 4*2;
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #10
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testFailingHashSecondMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(new NirvanaOutputList());
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockFailingMatchStub.class);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException etex) {
		// good!
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
}
Example #11
Source File: CachedMatchTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testHash4MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 1;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildSecondCachedJoinDriver<Record, Record, Record> testTask = new BuildSecondCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockMatchStub.class, 3);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
Example #12
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testHash5MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
Example #13
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testHash4MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 1;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
Example #14
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testHash3MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
Example #15
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testHash2MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 20;
	int valCnt2 = 1;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
Example #16
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testFailingHashFirstMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(new NirvanaOutputList());
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);
	
	JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		testDriver(testTask, MockFailingMatchStub.class);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException etex) {
		// good!
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
}
Example #17
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testMergeMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, true));
	
	try {
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
Example #18
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSortSecondMatchTask() {

	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
Example #19
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSortFirstMatchTask() {

	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInput(new UniformRecordGenerator(keyCnt2, valCnt2, true));
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
Example #20
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSortSecondMatchTask() {

	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInput(new UniformRecordGenerator(keyCnt1, valCnt1, true));
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
Example #21
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSortBoth4MatchTask() {

	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 1;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
Example #22
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSortBoth3MatchTask() {

	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
Example #23
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSortBoth2MatchTask() {

	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 20;
	int valCnt2 = 1;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
	
}
Example #24
Source File: JoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSortBoth1MatchTask() {
	final int keyCnt1 = 20;
	final int valCnt1 = 1;
	
	final int keyCnt2 = 10;
	final int valCnt2 = 2;
	
	setOutput(this.outList);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.INNER_MERGE);
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final JoinDriver<Record, Record, Record> testTask = new JoinDriver<>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockMatchStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertTrue("Resultset size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
}
Example #25
Source File: CachedMatchTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testFailingHashSecondMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(new NirvanaOutputList());
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildSecondCachedJoinDriver<Record, Record, Record> testTask = new BuildSecondCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockFailingMatchStub.class, 3);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException etex) {
		// good!
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
}
Example #26
Source File: CachedMatchTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testFailingHashFirstMatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(new NirvanaOutputList());
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockFailingMatchStub.class, 3);
		Assert.fail("Function exception was not forwarded.");
	} catch (ExpectedTestException etex) {
		// good!
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
}
Example #27
Source File: CachedMatchTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testHash5MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockMatchStub.class, 3);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
Example #28
Source File: CachedMatchTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testHash4MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 1;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildSecondCachedJoinDriver<Record, Record, Record> testTask = new BuildSecondCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockMatchStub.class, 3);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
Example #29
Source File: CachedMatchTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testHash3MatchTask() {
	int keyCnt1 = 20;
	int valCnt1 = 1;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
	addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED);
	getTaskConfig().setRelativeMemoryDriver(1.0f);
	
	BuildFirstCachedJoinDriver<Record, Record, Record> testTask = new BuildFirstCachedJoinDriver<Record, Record, Record>();
	
	try {
		testResettableDriver(testTask, MockMatchStub.class, 3);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Test caused an exception.");
	}
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2);
	Assert.assertEquals("Wrong result set size.", expCnt, this.outList.size());
	this.outList.clear();
}
Example #30
Source File: CoGroupTaskTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSortBoth1CoGroupTask() {
	int keyCnt1 = 100;
	int valCnt1 = 2;
	
	int keyCnt2 = 200;
	int valCnt2 = 1;
	
	final int expCnt = valCnt1*valCnt2*Math.min(keyCnt1, keyCnt2) + 
		(keyCnt1 > keyCnt2 ? (keyCnt1 - keyCnt2) * valCnt1 : (keyCnt2 - keyCnt1) * valCnt2);
	
	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(RecordPairComparatorFactory.get());
	getTaskConfig().setDriverStrategy(DriverStrategy.CO_GROUP);
	
	final CoGroupDriver<Record, Record, Record> testTask = new CoGroupDriver<Record, Record, Record>();
	
	try {
		addInputSorted(new UniformRecordGenerator(keyCnt1, valCnt1, false), this.comparator1.duplicate());
		addInputSorted(new UniformRecordGenerator(keyCnt2, valCnt2, false), this.comparator2.duplicate());
		testDriver(testTask, MockCoGroupStub.class);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("The test caused an exception.");
	}
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}