Java Code Examples for org.apache.flink.runtime.operators.testutils.types.IntPair#getKey()

@Override
public void join(IntPair rec1, Tuple2<Integer, String> rec2, Collector<Tuple2<Integer, String>> out) throws Exception
{
	final int k = rec1.getKey();
	final int v = rec1.getValue(); 
	
	final Integer key = rec2.f0;
	final String value = rec2.f1;

	Assert.assertTrue("Key does not match for matching IntPair Tuple combination.", k == key);
	
	Collection<TupleIntPairMatch> matches = this.toRemoveFrom.get(key);
	if (matches == null) {
		Assert.fail("Match " + key + " - " + v + ":" + value + " is unexpected.");
	}
	
	Assert.assertTrue("Produced match was not contained: " + key + " - " + v + ":" + value,
		matches.remove(new TupleIntPairMatch(v, value)));
	
	if (matches.isEmpty()) {
		this.toRemoveFrom.remove(key);
	}
}
 

@Override
public void putNormalizedKey(IntPair record, MemorySegment target, int offset, int len) {
	// see IntValue for a documentation of the logic
	final int value = record.getKey() - Integer.MIN_VALUE;

	if (len == 4) {
		target.putIntBigEndian(offset, value);
	} else if (len <= 0) {
	} else if (len < 4) {
		for (int i = 0; len > 0; len--, i++) {
			target.put(offset + i, (byte) ((value >>> ((3 - i) << 3)) & 0xff));
		}
	} else {
		target.putIntBigEndian(offset, value);
		for (int i = 4; i < len; i++) {
			target.put(offset + i, (byte) 0);
		}
	}
}
 

@Override
public void putNormalizedKey(IntPair record, MemorySegment target, int offset, int len) {
	// see IntValue for a documentation of the logic
	final int value = record.getKey() - Integer.MIN_VALUE;

	if (len == 4) {
		target.putIntBigEndian(offset, value);
	} else if (len <= 0) {
	} else if (len < 4) {
		for (int i = 0; len > 0; len--, i++) {
			target.put(offset + i, (byte) ((value >>> ((3 - i) << 3)) & 0xff));
		}
	} else {
		target.putIntBigEndian(offset, value);
		for (int i = 4; i < len; i++) {
			target.put(offset + i, (byte) 0);
		}
	}
}
 

public static Map<Integer, Collection<Integer>> collectIntPairData(MutableObjectIterator<IntPair> iter)
throws Exception
{
	Map<Integer, Collection<Integer>> map = new HashMap<>();
	IntPair pair = new IntPair();
	
	while ((pair = iter.next(pair)) != null) {

		final int key = pair.getKey();
		final int value = pair.getValue();
		if (!map.containsKey(key)) {
			map.put(key, new ArrayList<Integer>());
		}

		Collection<Integer> values = map.get(key);
		values.add(value);
	}

	return map;
}
 

@Override
public void join(IntPair rec1, Tuple2<Integer, String> rec2, Collector<Tuple2<Integer, String>> out) throws Exception
{
	final int k = rec1.getKey();
	final int v = rec1.getValue(); 
	
	final Integer key = rec2.f0;
	final String value = rec2.f1;

	Assert.assertTrue("Key does not match for matching IntPair Tuple combination.", k == key);
	
	Collection<TupleIntPairMatch> matches = this.toRemoveFrom.get(key);
	if (matches == null) {
		Assert.fail("Match " + key + " - " + v + ":" + value + " is unexpected.");
	}
	
	Assert.assertTrue("Produced match was not contained: " + key + " - " + v + ":" + value,
		matches.remove(new TupleIntPairMatch(v, value)));
	
	if (matches.isEmpty()) {
		this.toRemoveFrom.remove(key);
	}
}
 

@Test
public void testReset() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the buffer full with the first set of records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	sorter.reset();
	
	// write a second sequence of records. since the values are of fixed length, we must be able to write an equal number
	generator.reset();
	
	// write the buffer full with the first set of records
	int num2 = -1;
	do {
		generator.next(record);
		num2++;
	}
	while (sorter.write(record) && num2 < 3354624);
	
	Assert.assertEquals("The number of records written after the reset was not the same as before.", num, num2);
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = 0;
	while (i < num) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i++);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Override
public boolean equalToReference(IntPair candidate) {
	return candidate.getKey() == this.reference;
}
 

@Override
public void setReference(IntPair reference) {
	this.reference = reference.getKey();	
}
 

@Test
public void testReset() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the buffer full with the first set of records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	sorter.reset();
	
	// write a second sequence of records. since the values are of fixed length, we must be able to write an equal number
	generator.reset();
	
	// write the buffer full with the first set of records
	int num2 = -1;
	do {
		generator.next(record);
		num2++;
	}
	while (sorter.write(record) && num2 < 3354624);
	
	Assert.assertEquals("The number of records written after the reset was not the same as before.", num, num2);
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = 0;
	while (i < num) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i++);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Override
public boolean equalToReference(IntPair candidate) {
	return candidate.getKey() == this.reference;
}
 

@Override
public void setReference(IntPair reference) {
	this.reference = reference.getKey();	
}
 

@Test
public void testWriteAndIterator() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record));
	
	// re-read the records
	generator.reset();
	
	MutableObjectIterator<IntPair> iter = sorter.getIterator();
	IntPair readTarget = new IntPair();
	int count = 0;
	
	while ((readTarget = iter.next(readTarget)) != null) {
		count++;
		
		generator.next(record);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	Assert.assertEquals("Incorrect number of records", num, count);
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
public void testSpillingSortWithIntermediateMergeIntPair() {
	try {
		// amount of pairs
		final int PAIRS = 50000000;

		// comparator
		final RandomIntPairGenerator generator = new RandomIntPairGenerator(12345678, PAIRS);
		
		final TypeSerializerFactory<IntPair> serializerFactory = new IntPairSerializer.IntPairSerializerFactory();
		final TypeComparator<IntPair> comparator = new TestData.IntPairComparator();
		
		// merge iterator
		LOG.debug("Initializing sortmerger...");
		
		Sorter<IntPair> merger = new UnilateralSortMerger<IntPair>(this.memoryManager, this.ioManager, 
				generator, this.parentTask, serializerFactory, comparator, (double)64/78, 4, 0.7f,
				true /*use large record handler*/, true);

		// emit data
		LOG.debug("Emitting data...");
		
		// check order
		MutableObjectIterator<IntPair> iterator = merger.getIterator();
		
		LOG.debug("Checking results...");
		int pairsRead = 1;
		int nextStep = PAIRS / 20;

		IntPair rec1 = new IntPair();
		IntPair rec2 = new IntPair();
		
		Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
		
		while ((rec2 = iterator.next(rec2)) != null) {
			final int k1 = rec1.getKey();
			final int k2 = rec2.getKey();
			pairsRead++;
			
			Assert.assertTrue(k1 - k2 <= 0); 
			
			IntPair tmp = rec1;
			rec1 = rec2;
			rec2 = tmp;
			
			// log
			if (pairsRead == nextStep) {
				nextStep += PAIRS / 20;
			}
		}
		Assert.assertEquals("Not all pairs were read back in.", PAIRS, pairsRead);
		merger.close();
		testSuccess = true;
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

/**
 * The swap test fills the sort buffer and swaps all elements such that they are
 * backwards. It then resets the generator, goes backwards through the buffer
 * and compares for equality.
 */
@Test
public void testSwap() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	// swap the records
	int start = 0, end = num - 1;
	while (start < end) {
		sorter.swap(start++, end--);
	}
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = num - 1;
	while (i >= 0) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i--);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Override
public boolean equalToReference(IntPair candidate) {
	return this.reference == candidate.getKey();
}
 

@Override
public void setReference(IntPair reference) {
	this.reference = reference.getKey();	
}
 

@Override
public boolean equalToReference(IntPair candidate) {
	return candidate.getKey() == this.reference;
}
 

@Test
public void testSort() throws Exception {
	final int NUM_RECORDS = 559273;
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < NUM_RECORDS);
	
	QuickSort qs = new QuickSort();
	qs.sort(sorter);
	
	MutableObjectIterator<IntPair> iter = sorter.getIterator();
	IntPair readTarget = new IntPair();
	
	int current;
	int last;
	
	iter.next(readTarget);
	last = readTarget.getKey();
	
	while ((readTarget = iter.next(readTarget)) != null) {
		current = readTarget.getKey();
		
		final int cmp = last - current;
		if (cmp > 0) {
			Assert.fail("Next key is not larger or equal to previous key.");
		}
		last = current;
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Override
public boolean equalToReference(IntPair candidate) {
	return this.reference == candidate.getKey();
}
 

@Test
public void testSort() throws Exception {
	final int NUM_RECORDS = 559273;
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < NUM_RECORDS);
	
	QuickSort qs = new QuickSort();
	qs.sort(sorter);
	
	MutableObjectIterator<IntPair> iter = sorter.getIterator();
	IntPair readTarget = new IntPair();
	
	int current;
	int last;
	
	iter.next(readTarget);
	last = readTarget.getKey();
	
	while ((readTarget = iter.next(readTarget)) != null) {
		current = readTarget.getKey();
		
		final int cmp = last - current;
		if (cmp > 0) {
			Assert.fail("Next key is not larger or equal to previous key.");
		}
		last = current;
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}