org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator Java Examples

The following examples show how to use org.apache.flink.runtime.operators.testutils.UniformIntTupleGenerator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RightOuterJoinTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void testHashRightOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {

		setOutput(this.outList, this.serializer);
		addDriverComparator(this.comparator1);
		addDriverComparator(this.comparator2);
		getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
		getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
		getTaskConfig().setRelativeMemoryDriver(hash_frac);

		setNumFileHandlesForSort(4);

		final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

		addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer);
		addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer);
		testDriver(testTask, MockJoinStub.class);

		final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

		Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);

		this.outList.clear();
	}
Example #2
Source File: LeftOuterJoinTaskExternalITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testExternalHashLeftOuterJoinTask() throws Exception {

	final int keyCnt1 = 65536;
	final int valCnt1 = 8;

	final int keyCnt2 = 32768;
	final int valCnt2 = 8;

	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);

	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #3
Source File: AbstractOuterJoinTaskTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private void testSortBothOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {
	setOutput(this.outList, this.serializer);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);
	
	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
	
	Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
}
Example #4
Source File: AbstractOuterJoinTaskTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test(expected = ExpectedTestException.class)
public void testFailingOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
	
	testDriver(testTask, MockFailingJoinStub.class);
}
Example #5
Source File: AbstractOuterJoinTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test(expected = ExpectedTestException.class)
public void testFailingOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
	
	testDriver(testTask, MockFailingJoinStub.class);
}
Example #6
Source File: LeftOuterJoinTaskTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test(expected = ExpectedTestException.class)
public void testFailingHashLeftOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
Example #7
Source File: LeftOuterJoinTaskTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private void testHashLeftOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {

		setOutput(this.outList, this.serializer);
		addDriverComparator(this.comparator1);
		addDriverComparator(this.comparator2);
		getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
		getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
		getTaskConfig().setRelativeMemoryDriver(hash_frac);

		final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

		addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer);
		addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer);
		testDriver(testTask, MockJoinStub.class);

		final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

		Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);

		this.outList.clear();
	}
Example #8
Source File: AbstractOuterJoinTaskExternalITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testExternalSortOuterJoinTask() throws Exception {
	final int keyCnt1 = 16384 * 4;
	final int valCnt1 = 2;
	
	final int keyCnt2 = 8192;
	final int valCnt2 = 4 * 2;
	
	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
	
	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortStrategy());
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #9
Source File: RightOuterJoinTaskExternalITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testExternalHashRightOuterJoinTask() throws Exception {

	final int keyCnt1 = 32768;
	final int valCnt1 = 8;

	final int keyCnt2 = 65536;
	final int valCnt2 = 8;

	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);

	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #10
Source File: RightOuterJoinTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void testHashRightOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {

		setOutput(this.outList, this.serializer);
		addDriverComparator(this.comparator1);
		addDriverComparator(this.comparator2);
		getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
		getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
		getTaskConfig().setRelativeMemoryDriver(hash_frac);

		setNumFileHandlesForSort(4);

		final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

		addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer);
		addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer);
		testDriver(testTask, MockJoinStub.class);

		final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

		Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);

		this.outList.clear();
	}
Example #11
Source File: RightOuterJoinTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test(expected = ExpectedTestException.class)
public void testFailingHashRightOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
	setNumFileHandlesForSort(4);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
Example #12
Source File: AbstractOuterJoinTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void testSortBothOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {
	setOutput(this.outList, this.serializer);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);
	
	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
	
	Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
}
Example #13
Source File: RightOuterJoinTaskExternalITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testExternalHashRightOuterJoinTask() throws Exception {

	final int keyCnt1 = 32768;
	final int valCnt1 = 8;

	final int keyCnt2 = 65536;
	final int valCnt2 = 8;

	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);

	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #14
Source File: RightOuterJoinTaskTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private void testHashRightOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {

		setOutput(this.outList, this.serializer);
		addDriverComparator(this.comparator1);
		addDriverComparator(this.comparator2);
		getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
		getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
		getTaskConfig().setRelativeMemoryDriver(hash_frac);

		setNumFileHandlesForSort(4);

		final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

		addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer);
		addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer);
		testDriver(testTask, MockJoinStub.class);

		final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

		Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);

		this.outList.clear();
	}
Example #15
Source File: RightOuterJoinTaskTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test(expected = ExpectedTestException.class)
public void testFailingHashRightOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
	setNumFileHandlesForSort(4);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
Example #16
Source File: LeftOuterJoinTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test(expected = ExpectedTestException.class)
public void testFailingHashLeftOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
Example #17
Source File: LeftOuterJoinTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void testHashLeftOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {

		setOutput(this.outList, this.serializer);
		addDriverComparator(this.comparator1);
		addDriverComparator(this.comparator2);
		getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
		getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
		getTaskConfig().setRelativeMemoryDriver(hash_frac);

		final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

		addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer);
		addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer);
		testDriver(testTask, MockJoinStub.class);

		final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

		Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);

		this.outList.clear();
	}
Example #18
Source File: AbstractOuterJoinTaskExternalITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testExternalSortOuterJoinTask() throws Exception {
	final int keyCnt1 = 16384 * 4;
	final int valCnt1 = 2;
	
	final int keyCnt2 = 8192;
	final int valCnt2 = 4 * 2;
	
	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
	
	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortStrategy());
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #19
Source File: AbstractOuterJoinTaskExternalITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testExternalSortOuterJoinTask() throws Exception {
	final int keyCnt1 = 16384 * 4;
	final int valCnt1 = 2;
	
	final int keyCnt2 = 8192;
	final int valCnt2 = 4 * 2;
	
	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
	
	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortStrategy());
	getTaskConfig().setRelativeMemoryDriver(bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);
	
	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #20
Source File: LeftOuterJoinTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void testHashLeftOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {

		setOutput(this.outList, this.serializer);
		addDriverComparator(this.comparator1);
		addDriverComparator(this.comparator2);
		getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
		getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
		getTaskConfig().setRelativeMemoryDriver(hash_frac);

		final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

		addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer);
		addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer);
		testDriver(testTask, MockJoinStub.class);

		final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

		Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);

		this.outList.clear();
	}
Example #21
Source File: LeftOuterJoinTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test(expected = ExpectedTestException.class)
public void testFailingHashLeftOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
Example #22
Source File: LeftOuterJoinTaskExternalITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testExternalHashLeftOuterJoinTask() throws Exception {

	final int keyCnt1 = 65536;
	final int valCnt1 = 8;

	final int keyCnt2 = 32768;
	final int valCnt2 = 8;

	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);

	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #23
Source File: AbstractOuterJoinTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void testSortBothOuterJoinTask(int keyCnt1, int valCnt1, int keyCnt2, int valCnt2) throws Exception {
	setOutput(this.outList, this.serializer);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), this.serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);
	
	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
	
	Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
}
Example #24
Source File: RightOuterJoinTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test(expected = ExpectedTestException.class)
public void testFailingHashRightOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;

	int keyCnt2 = 20;
	int valCnt2 = 20;

	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);
	setNumFileHandlesForSort(4);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);

	testDriver(testTask, MockFailingJoinStub.class);
}
Example #25
Source File: AbstractOuterJoinTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test(expected = ExpectedTestException.class)
public void testFailingOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInput(new UniformIntTupleGenerator(keyCnt1, valCnt1, true), this.serializer);
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
	
	testDriver(testTask, MockFailingJoinStub.class);
}
Example #26
Source File: RightOuterJoinTaskExternalITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testExternalHashRightOuterJoinTask() throws Exception {

	final int keyCnt1 = 32768;
	final int valCnt1 = 8;

	final int keyCnt2 = 65536;
	final int valCnt2 = 8;

	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);

	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #27
Source File: LeftOuterJoinTaskExternalITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testExternalHashLeftOuterJoinTask() throws Exception {

	final int keyCnt1 = 65536;
	final int valCnt1 = 8;

	final int keyCnt2 = 32768;
	final int valCnt2 = 8;

	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);

	setOutput(this.output);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.LEFT_HYBRIDHASH_BUILD_SECOND);
	getTaskConfig().setRelativeMemoryDriver(hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), serializer, this.comparator1.duplicate());
	addInputSorted(new UniformIntTupleGenerator(keyCnt2, valCnt2, false), serializer, this.comparator2.duplicate());
	testDriver(testTask, MockJoinStub.class);

	Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
Example #28
Source File: RightOuterJoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCancelRightOuterJoinTaskWhileBuilding() throws Exception {
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);
	addInput(new UniformIntTupleGenerator(100, 100, true), this.serializer);

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {
		@Override
		public void run() {
			try {
				testDriver(testTask, MockJoinStub.class);
			} catch (Throwable t) {
				error.set(t);
			}
		}
	};
	taskRunner.start();

	Thread.sleep(1000);

	cancel();
	taskRunner.join(60000);

	assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());

	final Throwable taskError = error.get();
	if (taskError != null) {
		fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
	}
}
Example #29
Source File: AbstractOuterJoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSortFirstOuterJoinTask() throws Exception {
	int keyCnt1 = 20;
	int valCnt1 = 20;
	
	int keyCnt2 = 20;
	int valCnt2 = 20;
	
	setOutput(this.outList, this.serializer);
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(this.getSortDriverStrategy());
	getTaskConfig().setRelativeMemoryDriver(this.bnljn_frac);
	setNumFileHandlesForSort(4);
	
	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();
	
	addInputSorted(new UniformIntTupleGenerator(keyCnt1, valCnt1, false), this.serializer, this.comparator1.duplicate());
	addInput(new UniformIntTupleGenerator(keyCnt2, valCnt2, true), this.serializer);
	testDriver(testTask, MockJoinStub.class);
	
	final int expCnt = calculateExpectedCount(keyCnt1, valCnt1, keyCnt2, valCnt2);
	
	Assert.assertTrue("Result set size was " + this.outList.size() + ". Expected was " + expCnt, this.outList.size() == expCnt);
	
	this.outList.clear();
}
Example #30
Source File: RightOuterJoinTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCancelRightOuterJoinTaskWhileProbing() throws Exception {
	setOutput(new DiscardingOutputCollector<Tuple2<Integer, Integer>>());
	addDriverComparator(this.comparator1);
	addDriverComparator(this.comparator2);
	getTaskConfig().setDriverPairComparator(new RuntimePairComparatorFactory());
	getTaskConfig().setDriverStrategy(DriverStrategy.RIGHT_HYBRIDHASH_BUILD_FIRST);
	getTaskConfig().setRelativeMemoryDriver(this.hash_frac);

	final AbstractOuterJoinDriver<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> testTask = getOuterJoinDriver();

	addInput(new UniformIntTupleGenerator(1, 1, true), this.serializer);
	addInput(new DelayingIterator<>(new InfiniteIntTupleIterator(), 100), this.serializer);

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final Thread taskRunner = new Thread("Task runner for testCancelOuterJoinTaskWhileSort1()") {
		@Override
		public void run() {
			try {
				testDriver(testTask, MockJoinStub.class);
			} catch (Throwable t) {
				error.set(t);
			}
		}
	};
	taskRunner.start();

	Thread.sleep(1000);

	cancel();
	taskRunner.join(60000);

	assertFalse("Task thread did not finish within 60 seconds", taskRunner.isAlive());

	final Throwable taskError = error.get();
	if (taskError != null) {
		fail("Error in task while canceling:\n" + Throwables.getStackTraceAsString(taskError));
	}
}