org.apache.flink.runtime.operators.testutils.TestData Java Examples

@SuppressWarnings({"unchecked", "rawtypes"})
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();

	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);

	this.recordBuildSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordProbeSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordBuildSideComparator = TestData.getIntIntTupleComparator();
	this.recordProbeSideComparator = TestData.getIntIntTupleComparator();
	this.pactRecordComparator = new GenericPairComparator(this.recordBuildSideComparator, this.recordProbeSideComparator);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
	this.ioManager = new IOManagerAsync();
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();
	
	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);
	
	this.pairSerializer = new IntPairSerializer();
	this.pairComparator = new TestData.IntPairComparator();
	this.pairRecordPairComparator = new IntPairTuplePairComparator();
	this.recordPairPairComparator = new TupleIntPairPairComparator();
	
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 

@SuppressWarnings({"unchecked", "rawtypes"})
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();

	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);

	this.recordBuildSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordProbeSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordBuildSideComparator = TestData.getIntIntTupleComparator();
	this.recordProbeSideComparator = TestData.getIntIntTupleComparator();
	this.pactRecordComparator = new GenericPairComparator(this.recordBuildSideComparator, this.recordProbeSideComparator);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
	this.ioManager = new IOManagerAsync();
}
 

@SuppressWarnings({"unchecked", "rawtypes"})
@Before
public void beforeTest() {
	this.recordSerializer = TestData.getIntStringTupleSerializer();

	this.record1Comparator = TestData.getIntStringTupleComparator();
	this.record2Comparator = TestData.getIntStringTupleComparator();
	this.recordPairComparator = new GenericPairComparator(this.record1Comparator, this.record2Comparator);

	this.recordBuildSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordProbeSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordBuildSideComparator = TestData.getIntIntTupleComparator();
	this.recordProbeSideComparator = TestData.getIntIntTupleComparator();
	this.pactRecordComparator = new GenericPairComparator(this.recordBuildSideComparator, this.recordProbeSideComparator);

	this.memoryManager = MemoryManagerBuilder
		.newBuilder()
		.setMemorySize(MEMORY_SIZE)
		.setPageSize(PAGE_SIZE)
		.build();
	this.ioManager = new IOManagerAsync();
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
	
	this.serializerFactory1 = TestData.getIntStringTupleSerializerFactory();
	this.comparator1 = TestData.getIntStringTupleComparator();

	this.serializerFactory2 = TestData.getIntIntTupleSerializerFactory();
	this.comparator2 = TestData.getIntIntTupleComparator();
}
 

/**
 * The compare test creates a sorted stream, writes it to the buffer and
 * compares random elements. It expects that earlier elements are lower than later
 * ones.
 */
@Test
public void testCompare() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
	TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.SORTED,
		ValueMode.RANDOM_LENGTH);
	
	// write the records
	Tuple2<Integer, String> record = new Tuple2<>();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record));
	
	// compare random elements
	Random rnd = new Random(SEED << 1);
	for (int i = 0; i < 2 * num; i++) {
		int pos1 = rnd.nextInt(num);
		int pos2 = rnd.nextInt(num);
		
		int cmp = sorter.compare(pos1, pos2);
		
		if (pos1 < pos2) {
			Assert.assertTrue(cmp <= 0);
		}
		else {
			Assert.assertTrue(cmp >= 0);
		}
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
public void testBuildSecond() {
	try {
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		
		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		final FlatJoinFunction matcher = new NoOpMatcher();
		
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
		
		long start = System.nanoTime();
		
		// compare with iterator values
		ReusingBuildSecondHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
				new ReusingBuildSecondHashJoinIterator<>(
					input1, input2, this.serializer1.getSerializer(), this.comparator1, 
					this.serializer2.getSerializer(), this.comparator2, this.pairComparator11,
					this.memoryManager, this.ioManager, this.parentTask, 1, false, false, true);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();
		
		long elapsed = System.nanoTime() - start;
		double msecs = elapsed / (1000 * 1000);
		
		System.out.println("Hash Build Second took " + msecs + " msecs.");
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 

@Test
public void test() throws Exception {
	final TupleGenerator generator1 =
			new TupleGenerator(SEED1, 500, 4096, KeyMode.SORTED, ValueMode.RANDOM_LENGTH);
	final TupleGenerator generator2 =
			new TupleGenerator(SEED2, 500, 2048, KeyMode.SORTED, ValueMode.RANDOM_LENGTH);

	final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_FIRST_SIZE);
	final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_SECOND_SIZE);

	// collect expected data
	final Map<Integer, Collection<Match>> expectedMatchesMap = matchValues(
			collectData(input1), collectData(input2));

	// reset the generators
	generator1.reset();
	generator2.reset();
	input1.reset();
	input2.reset();

	StreamOperator operator = getOperator();

	match(expectedMatchesMap, transformToBinary(join(operator, input1, input2)));

	// assert that each expected match was seen
	for (Map.Entry<Integer, Collection<Match>> entry : expectedMatchesMap.entrySet()) {
		Assert.assertTrue("Collection for key " + entry.getKey() + " is not empty", entry.getValue().isEmpty());
	}
}
 

@SuppressWarnings({"unchecked", "rawtypes"})
@Before
public void beforeTest() {
	this.recordBuildSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordProbeSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordBuildSideComparator = TestData.getIntIntTupleComparator();
	this.recordProbeSideComparator = TestData.getIntIntTupleComparator();
	this.pactRecordComparator = new GenericPairComparator(this.recordBuildSideComparator, this.recordProbeSideComparator);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
	this.ioManager = new IOManagerAsync();
}
 

@Test
public void testCombine() throws Exception
{
	int noKeys = 100;
	int noKeyCnt = 10000;

	TestData.MockTuple2Reader<Tuple2<Integer, Integer>> reader = TestData.getIntIntTupleReader();

	LOG.debug("initializing sortmerger");
	
	TestCountCombiner comb = new TestCountCombiner();
	
	Sorter<Tuple2<Integer, Integer>> merger = new CombiningUnilateralSortMerger<>(comb,
			this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory2, this.comparator2,
			0.25, 64, 0.7f, true /* use large record handler */, false);

	final Tuple2<Integer, Integer> rec = new Tuple2<>();
	rec.setField(1, 1);
	
	for (int i = 0; i < noKeyCnt; i++) {
		for (int j = 0; j < noKeys; j++) {
			rec.setField(j, 0);
			reader.emit(rec);
		}
	}
	reader.close();
	
	MutableObjectIterator<Tuple2<Integer, Integer>> iterator = merger.getIterator();

	Iterator<Integer> result = getReducingIterator(iterator, serializerFactory2.getSerializer(), comparator2.duplicate());
	while (result.hasNext()) {
		Assert.assertEquals(noKeyCnt, result.next().intValue());
	}
	
	merger.close();
	
	// if the combiner was opened, it must have been closed
	Assert.assertTrue(comb.opened == comb.closed);
}
 

/**
 * The compare test creates a sorted stream, writes it to the buffer and
 * compares random elements. It expects that earlier elements are lower than later
 * ones.
 */
@Test
public void testCompare() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
	TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.SORTED,
		ValueMode.RANDOM_LENGTH);
	
	// write the records
	Tuple2<Integer, String> record = new Tuple2<>();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record));
	
	// compare random elements
	Random rnd = new Random(SEED << 1);
	for (int i = 0; i < 2 * num; i++) {
		int pos1 = rnd.nextInt(num);
		int pos2 = rnd.nextInt(num);
		
		int cmp = sorter.compare(pos1, pos2);
		
		if (pos1 < pos2) {
			Assert.assertTrue(cmp <= 0);
		}
		else {
			Assert.assertTrue(cmp >= 0);
		}
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
public void testWriteAndIterator() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
	TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM,
		ValueMode.RANDOM_LENGTH);
	
	// write the records
	Tuple2<Integer, String> record = new Tuple2<>();
	do {
		generator.next(record);
	}
	while (sorter.write(record));
	
	// re-read the records
	generator.reset();
	MutableObjectIterator<Tuple2<Integer, String>> iter = sorter.getIterator();
	Tuple2<Integer, String> readTarget = new Tuple2<>();
	
	while ((readTarget = iter.next(readTarget)) != null) {
		generator.next(record);
		
		int rk = readTarget.f0;
		int gk = record.f0;
		
		String rv = readTarget.f1;
		String gv = record.f1;
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
public void testCombineSpilling() throws Exception {
	int noKeys = 100;
	int noKeyCnt = 10000;

	TestData.MockTuple2Reader<Tuple2<Integer, Integer>> reader = TestData.getIntIntTupleReader();

	LOG.debug("initializing sortmerger");
	
	TestCountCombiner comb = new TestCountCombiner();
	
	Sorter<Tuple2<Integer, Integer>> merger = new CombiningUnilateralSortMerger<>(comb,
			this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory2, this.comparator2,
			0.01, 64, 0.005f, true /* use large record handler */, true);

	final Tuple2<Integer, Integer> rec = new Tuple2<>();
	rec.setField(1, 1);
	
	for (int i = 0; i < noKeyCnt; i++) {
		for (int j = 0; j < noKeys; j++) {
			rec.setField(j, 0);
			reader.emit(rec);
		}
	}
	reader.close();
	
	MutableObjectIterator<Tuple2<Integer, Integer>> iterator = merger.getIterator();

	Iterator<Integer> result = getReducingIterator(iterator, serializerFactory2.getSerializer(), comparator2.duplicate());
	while (result.hasNext()) {
		Assert.assertEquals(noKeyCnt, result.next().intValue());
	}
	
	merger.close();
	
	// if the combiner was opened, it must have been closed
	Assert.assertTrue(comb.opened == comb.closed);
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
	
	this.pactRecordSerializer = TestData.getIntStringTupleSerializerFactory();
	this.pactRecordComparator = TestData.getIntStringTupleComparator();
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.serializer1 = TestData.getIntStringTupleSerializer();
	this.serializer2 = TestData.getIntStringTupleSerializer();
	this.comparator1 = TestData.getIntStringTupleComparator();
	this.comparator2 = TestData.getIntStringTupleComparator();
	this.pairComparator = new GenericPairComparator(this.comparator1, this.comparator2);
}
 

@Test
public void testCombine() throws Exception
{
	int noKeys = 100;
	int noKeyCnt = 10000;

	TestData.MockTuple2Reader<Tuple2<Integer, Integer>> reader = TestData.getIntIntTupleReader();

	LOG.debug("initializing sortmerger");
	
	TestCountCombiner comb = new TestCountCombiner();
	
	Sorter<Tuple2<Integer, Integer>> merger = new CombiningUnilateralSortMerger<>(comb,
			this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory2, this.comparator2,
			0.25, 64, 0.7f, true /* use large record handler */, false);

	final Tuple2<Integer, Integer> rec = new Tuple2<>();
	rec.setField(1, 1);
	
	for (int i = 0; i < noKeyCnt; i++) {
		for (int j = 0; j < noKeys; j++) {
			rec.setField(j, 0);
			reader.emit(rec);
		}
	}
	reader.close();
	
	MutableObjectIterator<Tuple2<Integer, Integer>> iterator = merger.getIterator();

	Iterator<Integer> result = getReducingIterator(iterator, serializerFactory2.getSerializer(), comparator2.duplicate());
	while (result.hasNext()) {
		Assert.assertEquals(noKeyCnt, result.next().intValue());
	}
	
	merger.close();
	
	// if the combiner was opened, it must have been closed
	Assert.assertTrue(comb.opened == comb.closed);
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.serializer1 = TestData.getIntStringTupleSerializer();
	this.serializer2 = TestData.getIntStringTupleSerializer();
	this.comparator1 = TestData.getIntStringTupleComparator();
	this.comparator2 = TestData.getIntStringTupleComparator();
	this.pairComparator = new GenericPairComparator(this.comparator1, this.comparator2);
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.serializer1 = TestData.getIntStringTupleSerializer();
	this.serializer2 = TestData.getIntStringTupleSerializer();
	this.comparator1 = TestData.getIntStringTupleComparator();
	this.comparator2 = TestData.getIntStringTupleComparator();
	this.pairComparator = new GenericPairComparator(this.comparator1, this.comparator2);
}
 

private MutableObjectIterator<Tuple2<Integer, Integer>> getProbeInput(final int numKeys,
																	  final int probeValsPerKey, final int repeatedValue1, final int repeatedValue2) {
	MutableObjectIterator<Tuple2<Integer, Integer>> probe1 = new UniformIntTupleGenerator(numKeys, probeValsPerKey, true);
	MutableObjectIterator<Tuple2<Integer, Integer>> probe2 = new TestData.ConstantIntIntTuplesIterator(repeatedValue1, 17, 5);
	MutableObjectIterator<Tuple2<Integer, Integer>> probe3 = new TestData.ConstantIntIntTuplesIterator(repeatedValue2, 23, 5);
	List<MutableObjectIterator<Tuple2<Integer, Integer>>> probes = new ArrayList<>();
	probes.add(probe1);
	probes.add(probe2);
	probes.add(probe3);
	return new UnionIterator<>(probes);
}
 

/**
 * The compare test creates a sorted stream, writes it to the buffer and
 * compares random elements. It expects that earlier elements are lower than later
 * ones.
 */
@Test
public void testCompare() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	NormalizedKeySorter<Tuple2<Integer, String>> sorter = newSortBuffer(memory);
	TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.SORTED,
		ValueMode.RANDOM_LENGTH);
	
	// write the records
	Tuple2<Integer, String> record = new Tuple2<>();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record));
	
	// compare random elements
	Random rnd = new Random(SEED << 1);
	for (int i = 0; i < 2 * num; i++) {
		int pos1 = rnd.nextInt(num);
		int pos2 = rnd.nextInt(num);
		
		int cmp = sorter.compare(pos1, pos2);
		
		if (pos1 < pos2) {
			Assert.assertTrue(cmp <= 0);
		}
		else {
			Assert.assertTrue(cmp >= 0);
		}
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
	this.serializer1 = TestData.getIntStringTupleSerializer();
	this.serializer2 = TestData.getIntStringTupleSerializer();
	this.comparator1 = TestData.getIntStringTupleComparator();
	this.comparator2 = TestData.getIntStringTupleComparator();
	this.pairComparator = new GenericPairComparator(comparator1, comparator2);
}
 

@Test
public void testCombine() throws Exception
{
	int noKeys = 100;
	int noKeyCnt = 10000;

	TestData.MockTuple2Reader<Tuple2<Integer, Integer>> reader = TestData.getIntIntTupleReader();

	LOG.debug("initializing sortmerger");
	
	TestCountCombiner comb = new TestCountCombiner();
	
	Sorter<Tuple2<Integer, Integer>> merger = new CombiningUnilateralSortMerger<>(comb,
			this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory2, this.comparator2,
			0.25, 64, 0.7f, true /* use large record handler */, false);

	final Tuple2<Integer, Integer> rec = new Tuple2<>();
	rec.setField(1, 1);
	
	for (int i = 0; i < noKeyCnt; i++) {
		for (int j = 0; j < noKeys; j++) {
			rec.setField(j, 0);
			reader.emit(rec);
		}
	}
	reader.close();
	
	MutableObjectIterator<Tuple2<Integer, Integer>> iterator = merger.getIterator();

	Iterator<Integer> result = getReducingIterator(iterator, serializerFactory2.getSerializer(), comparator2.duplicate());
	while (result.hasNext()) {
		Assert.assertEquals(noKeyCnt, result.next().intValue());
	}
	
	merger.close();
	
	// if the combiner was opened, it must have been closed
	Assert.assertTrue(comb.opened == comb.closed);
}
 

@Test
public void testBuildSecond() {
	try {
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		
		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		final FlatJoinFunction matcher = new NoOpMatcher();
		
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();
		
		long start = System.nanoTime();
		
		// compare with iterator values
		ReusingBuildSecondHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
				new ReusingBuildSecondHashJoinIterator<>(
					input1, input2, this.serializer1.getSerializer(), this.comparator1, 
					this.serializer2.getSerializer(), this.comparator2, this.pairComparator11,
					this.memoryManager, this.ioManager, this.parentTask, 1, false, false, true);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();
		
		long elapsed = System.nanoTime() - start;
		double msecs = elapsed / (1000 * 1000);
		
		System.out.println("Hash Build Second took " + msecs + " msecs.");
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 

@Test
public void testCombineSpillingDisableObjectReuse() throws Exception {
	int noKeys = 100;
	int noKeyCnt = 10000;

	TestData.MockTuple2Reader<Tuple2<Integer, Integer>> reader = TestData.getIntIntTupleReader();

	LOG.debug("initializing sortmerger");

	MaterializedCountCombiner comb = new MaterializedCountCombiner();

	// set maxNumFileHandles = 2 to trigger multiple channel merging
	Sorter<Tuple2<Integer, Integer>> merger = new CombiningUnilateralSortMerger<>(comb,
			this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory2, this.comparator2,
			0.01, 2, 0.005f, true /* use large record handler */, false);

	final Tuple2<Integer, Integer> rec = new Tuple2<>();

	for (int i = 0; i < noKeyCnt; i++) {
		rec.setField(i, 0);
		for (int j = 0; j < noKeys; j++) {
			rec.setField(j, 1);
			reader.emit(rec);
		}
	}
	reader.close();

	MutableObjectIterator<Tuple2<Integer, Integer>> iterator = merger.getIterator();
	Iterator<Integer> result = getReducingIterator(iterator, serializerFactory2.getSerializer(), comparator2.duplicate());
	while (result.hasNext()) {
		Assert.assertEquals(4950, result.next().intValue());
	}

	merger.close();
}
 

@SuppressWarnings({"unchecked", "rawtypes"})
@Before
public void beforeTest() {
	this.recordBuildSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordProbeSideAccesssor = TestData.getIntIntTupleSerializer();
	this.recordBuildSideComparator = TestData.getIntIntTupleComparator();
	this.recordProbeSideComparator = TestData.getIntIntTupleComparator();
	this.pactRecordComparator = new GenericPairComparator(this.recordBuildSideComparator, this.recordProbeSideComparator);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
	this.ioManager = new IOManagerAsync();
}
 

@Test
public void testBuildSecondWithHighNumberOfCommonKeys()
{
	// the size of the left and right inputs
	final int INPUT_1_SIZE = 200;
	final int INPUT_2_SIZE = 100;
	
	final int INPUT_1_DUPLICATES = 10;
	final int INPUT_2_DUPLICATES = 2000;
	final int DUPLICATE_KEY = 13;
	
	try {
		TupleGenerator generator1 = new TupleGenerator(SEED1, 500, 4096, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TupleGenerator generator2 = new TupleGenerator(SEED2, 500, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		
		final TestData.TupleGeneratorIterator gen1Iter = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator gen2Iter = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);
		
		final TestData.TupleConstantValueIterator const1Iter = new TestData.TupleConstantValueIterator(DUPLICATE_KEY, "LEFT String for Duplicate Keys", INPUT_1_DUPLICATES);
		final TestData.TupleConstantValueIterator const2Iter = new TestData.TupleConstantValueIterator(DUPLICATE_KEY, "RIGHT String for Duplicate Keys", INPUT_2_DUPLICATES);
		
		final List<MutableObjectIterator<Tuple2<Integer, String>>> inList1 = new ArrayList<>();
		inList1.add(gen1Iter);
		inList1.add(const1Iter);
		
		final List<MutableObjectIterator<Tuple2<Integer, String>>> inList2 = new ArrayList<>();
		inList2.add(gen2Iter);
		inList2.add(const2Iter);
		
		MutableObjectIterator<Tuple2<Integer, String>> input1 = new UnionIterator<>(inList1);
		MutableObjectIterator<Tuple2<Integer, String>> input2 = new UnionIterator<>(inList2);
		
		
		// collect expected data
		final Map<Integer, Collection<TupleMatch>> expectedMatchesMap = joinTuples(
				collectTupleData(input1),
				collectTupleData(input2));
		
		// re-create the whole thing for actual processing
		
		// reset the generators and iterators
		generator1.reset();
		generator2.reset();
		const1Iter.reset();
		const2Iter.reset();
		gen1Iter.reset();
		gen2Iter.reset();
		
		inList1.clear();
		inList1.add(gen1Iter);
		inList1.add(const1Iter);
		
		inList2.clear();
		inList2.add(gen2Iter);
		inList2.add(const2Iter);

		input1 = new UnionIterator<>(inList1);
		input2 = new UnionIterator<>(inList2);
		
		final TupleMatchRemovingJoin matcher = new TupleMatchRemovingJoin(expectedMatchesMap);
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();

		NonReusingBuildSecondHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
			new NonReusingBuildSecondHashJoinIterator<>(
				input1, input2, this.recordSerializer, this.record1Comparator, 
				this.recordSerializer, this.record2Comparator, this.recordPairComparator,
				this.memoryManager, ioManager, this.parentTask, 1.0, false, false, true);
		
		iterator.open();
		
		while (iterator.callWithNextKey(matcher, collector));
		
		iterator.close();

		// assert that each expected match was seen
		for (Entry<Integer, Collection<TupleMatch>> entry : expectedMatchesMap.entrySet()) {
			if (!entry.getValue().isEmpty()) {
				Assert.fail("Collection for key " + entry.getKey() + " is not empty");
			}
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 

@Test
public void testBuildFirstAndFullOuterJoin() {
	try {
		TestData.TupleGenerator generator1 = new TestData.TupleGenerator(SEED1, 500, 4096, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
		TestData.TupleGenerator generator2 = new TestData.TupleGenerator(SEED2, 1000, 2048, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);

		final TestData.TupleGeneratorIterator input1 = new TestData.TupleGeneratorIterator(generator1, INPUT_1_SIZE);
		final TestData.TupleGeneratorIterator input2 = new TestData.TupleGeneratorIterator(generator2, INPUT_2_SIZE);

		// collect expected data
		final Map<Integer, Collection<TupleMatch>> expectedMatchesMap = fullOuterJoinTuples(
			collectTupleData(input1),
			collectTupleData(input2));

		final FlatJoinFunction matcher = new TupleMatchRemovingJoin(expectedMatchesMap);
		final Collector<Tuple2<Integer, String>> collector = new DiscardingOutputCollector<>();

		// reset the generators
		generator1.reset();
		generator2.reset();
		input1.reset();
		input2.reset();

		// compare with iterator values
		ReusingBuildFirstHashJoinIterator<Tuple2<Integer, String>, Tuple2<Integer, String>, Tuple2<Integer, String>> iterator =
			new ReusingBuildFirstHashJoinIterator<>(
				input1, input2, this.recordSerializer, this.record1Comparator,
				this.recordSerializer, this.record2Comparator, this.recordPairComparator,
				this.memoryManager, ioManager, this.parentTask, 1.0, true, true, false);

		iterator.open();

		while (iterator.callWithNextKey(matcher, collector));

		iterator.close();

		// assert that each expected match was seen
		for (Entry<Integer, Collection<TupleMatch>> entry : expectedMatchesMap.entrySet()) {
			if (!entry.getValue().isEmpty()) {
				Assert.fail("Collection for key " + entry.getKey() + " is not empty");
			}
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail("An exception occurred during the test: " + e.getMessage());
	}
}
 

@Test
public void testWriteReadSmallRecords() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		
		int k1 = rec.f0;
		String v1 = rec.f1;
		
		int k2 = readRec.f0;
		String v2 = readRec.f1;
		
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testWriteAndReadLongRecords() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LONG_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_LONG; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_LONG; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		final int k1 = rec.f0;
		final String v1 = rec.f1;
		final int k2 = readRec.f0;
		final String v2 = readRec.f1;
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}
 

@Test
public void testWriteReadSmallRecords() throws Exception
{
	final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
	final FileIOChannel.ID channel = this.ioManager.createChannel();
	final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
	
	// create the writer output view
	List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
	// write a number of pairs
	final Tuple2<Integer, String> rec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.serialize(rec, outView);
	}
	this.memoryManager.release(outView.close());
	
	// create the reader input view
	memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
	final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel);
	final ChannelReaderInputView inView = new ChannelReaderInputView(reader, memory, outView.getBlockCount(), true);
	generator.reset();
	
	// read and re-generate all records and compare them
	final Tuple2<Integer, String> readRec = new Tuple2<>();
	for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
		generator.next(rec);
		serializer.deserialize(readRec, inView);
		
		int k1 = rec.f0;
		String v1 = rec.f1;
		
		int k2 = readRec.f0;
		String v2 = readRec.f1;
		
		Assert.assertTrue("The re-generated and the read record do not match.", k1 == k2 && v1.equals(v2));
	}
	
	this.memoryManager.release(inView.close());
	reader.deleteChannel();
}