Java Code Examples for org.apache.flink.runtime.operators.testutils.types.IntPair#getValue()

public static Map<Integer, Collection<Integer>> collectIntPairData(MutableObjectIterator<IntPair> iter)
throws Exception
{
	Map<Integer, Collection<Integer>> map = new HashMap<>();
	IntPair pair = new IntPair();
	
	while ((pair = iter.next(pair)) != null) {

		final int key = pair.getKey();
		final int value = pair.getValue();
		if (!map.containsKey(key)) {
			map.put(key, new ArrayList<Integer>());
		}

		Collection<Integer> values = map.get(key);
		values.add(value);
	}

	return map;
}
 

@Override
public void join(IntPair rec1, Tuple2<Integer, String> rec2, Collector<Tuple2<Integer, String>> out) throws Exception
{
	final int k = rec1.getKey();
	final int v = rec1.getValue(); 
	
	final Integer key = rec2.f0;
	final String value = rec2.f1;

	Assert.assertTrue("Key does not match for matching IntPair Tuple combination.", k == key);
	
	Collection<TupleIntPairMatch> matches = this.toRemoveFrom.get(key);
	if (matches == null) {
		Assert.fail("Match " + key + " - " + v + ":" + value + " is unexpected.");
	}
	
	Assert.assertTrue("Produced match was not contained: " + key + " - " + v + ":" + value,
		matches.remove(new TupleIntPairMatch(v, value)));
	
	if (matches.isEmpty()) {
		this.toRemoveFrom.remove(key);
	}
}
 

@Override
public void join(IntPair rec1, Tuple2<Integer, String> rec2, Collector<Tuple2<Integer, String>> out) throws Exception
{
	final int k = rec1.getKey();
	final int v = rec1.getValue(); 
	
	final Integer key = rec2.f0;
	final String value = rec2.f1;
	
	Assert.assertTrue("Key does not match for matching IntPair Tuple combination.", k == key); 
	
	Collection<TupleIntPairMatch> matches = this.toRemoveFrom.get(key);
	if (matches == null) {
		Assert.fail("Match " + key + " - " + v + ":" + value + " is unexpected.");
	}
	
	Assert.assertTrue("Produced match was not contained: " + key + " - " + v + ":" + value,
		matches.remove(new TupleIntPairMatch(v, value)));
	
	if (matches.isEmpty()) {
		this.toRemoveFrom.remove(key);
	}
}
 

@Override
public void join(IntPair rec1, Tuple2<Integer, String> rec2, Collector<Tuple2<Integer, String>> out) throws Exception
{
	final int k = rec1.getKey();
	final int v = rec1.getValue(); 
	
	final Integer key = rec2.f0;
	final String value = rec2.f1;

	Assert.assertTrue("Key does not match for matching IntPair Tuple combination.", k == key);
	
	Collection<TupleIntPairMatch> matches = this.toRemoveFrom.get(key);
	if (matches == null) {
		Assert.fail("Match " + key + " - " + v + ":" + value + " is unexpected.");
	}
	
	Assert.assertTrue("Produced match was not contained: " + key + " - " + v + ":" + value,
		matches.remove(new TupleIntPairMatch(v, value)));
	
	if (matches.isEmpty()) {
		this.toRemoveFrom.remove(key);
	}
}
 

@Override
public void join(IntPair rec1, Tuple2<Integer, String> rec2, Collector<Tuple2<Integer, String>> out) throws Exception
{
	final int k = rec1.getKey();
	final int v = rec1.getValue(); 
	
	final Integer key = rec2.f0;
	final String value = rec2.f1;
	
	Assert.assertTrue("Key does not match for matching IntPair Tuple combination.", k == key); 
	
	Collection<TupleIntPairMatch> matches = this.toRemoveFrom.get(key);
	if (matches == null) {
		Assert.fail("Match " + key + " - " + v + ":" + value + " is unexpected.");
	}
	
	Assert.assertTrue("Produced match was not contained: " + key + " - " + v + ":" + value,
		matches.remove(new TupleIntPairMatch(v, value)));
	
	if (matches.isEmpty()) {
		this.toRemoveFrom.remove(key);
	}
}
 

public static Map<Integer, Collection<Integer>> collectIntPairData(MutableObjectIterator<IntPair> iter)
throws Exception
{
	Map<Integer, Collection<Integer>> map = new HashMap<>();
	IntPair pair = new IntPair();
	
	while ((pair = iter.next(pair)) != null) {

		final int key = pair.getKey();
		final int value = pair.getValue();
		if (!map.containsKey(key)) {
			map.put(key, new ArrayList<Integer>());
		}

		Collection<Integer> values = map.get(key);
		values.add(value);
	}

	return map;
}
 

@Override
public void join(IntPair rec1, Tuple2<Integer, String> rec2, Collector<Tuple2<Integer, String>> out) throws Exception
{
	final int k = rec1.getKey();
	final int v = rec1.getValue(); 
	
	final Integer key = rec2.f0;
	final String value = rec2.f1;

	Assert.assertTrue("Key does not match for matching IntPair Tuple combination.", k == key);
	
	Collection<TupleIntPairMatch> matches = this.toRemoveFrom.get(key);
	if (matches == null) {
		Assert.fail("Match " + key + " - " + v + ":" + value + " is unexpected.");
	}
	
	Assert.assertTrue("Produced match was not contained: " + key + " - " + v + ":" + value,
		matches.remove(new TupleIntPairMatch(v, value)));
	
	if (matches.isEmpty()) {
		this.toRemoveFrom.remove(key);
	}
}
 

public static Map<Integer, Collection<Integer>> collectIntPairData(MutableObjectIterator<IntPair> iter)
throws Exception
{
	Map<Integer, Collection<Integer>> map = new HashMap<>();
	IntPair pair = new IntPair();
	
	while ((pair = iter.next(pair)) != null) {

		final int key = pair.getKey();
		final int value = pair.getValue();
		if (!map.containsKey(key)) {
			map.put(key, new ArrayList<Integer>());
		}

		Collection<Integer> values = map.get(key);
		values.add(value);
	}

	return map;
}
 

@Override
public void join(IntPair rec1, Tuple2<Integer, String> rec2, Collector<Tuple2<Integer, String>> out) throws Exception
{
	final int k = rec1.getKey();
	final int v = rec1.getValue(); 
	
	final Integer key = rec2.f0;
	final String value = rec2.f1;
	
	Assert.assertTrue("Key does not match for matching IntPair Tuple combination.", k == key); 
	
	Collection<TupleIntPairMatch> matches = this.toRemoveFrom.get(key);
	if (matches == null) {
		Assert.fail("Match " + key + " - " + v + ":" + value + " is unexpected.");
	}
	
	Assert.assertTrue("Produced match was not contained: " + key + " - " + v + ":" + value,
		matches.remove(new TupleIntPairMatch(v, value)));
	
	if (matches.isEmpty()) {
		this.toRemoveFrom.remove(key);
	}
}
 

@Test
public void testReset() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the buffer full with the first set of records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	sorter.reset();
	
	// write a second sequence of records. since the values are of fixed length, we must be able to write an equal number
	generator.reset();
	
	// write the buffer full with the first set of records
	int num2 = -1;
	do {
		generator.next(record);
		num2++;
	}
	while (sorter.write(record) && num2 < 3354624);
	
	Assert.assertEquals("The number of records written after the reset was not the same as before.", num, num2);
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = 0;
	while (i < num) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i++);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
public void testWriteAndIterator() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record));
	
	// re-read the records
	generator.reset();
	
	MutableObjectIterator<IntPair> iter = sorter.getIterator();
	IntPair readTarget = new IntPair();
	int count = 0;
	
	while ((readTarget = iter.next(readTarget)) != null) {
		count++;
		
		generator.next(record);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	Assert.assertEquals("Incorrect number of records", num, count);
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
	public void testWriteAndRead() throws Exception {
		final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
		final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
		
		FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
		RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
		
//		long startTime = System.currentTimeMillis();
		// write the records
		IntPair record = new IntPair();
		int num = -1;
		do {
			generator.next(record);
			num++;
		}
		while (sorter.write(record) && num < 3354624);
//		System.out.println("WRITE TIME " + (System.currentTimeMillis() - startTime));
		
		// re-read the records
		generator.reset();
		IntPair readTarget = new IntPair();
		
//		startTime = System.currentTimeMillis();
		int i = 0;
		while (i < num) {
			generator.next(record);
			readTarget = sorter.getRecord(readTarget, i++);
			
			int rk = readTarget.getKey();
			int gk = record.getKey();
			
			int rv = readTarget.getValue();
			int gv = record.getValue();
			
			if (gk != rk) {
				Assert.fail("The re-read key is wrong " + i);
			}
			if (gv != rv) {
				Assert.fail("The re-read value is wrong");
			}
		}
//		System.out.println("READ TIME " + (System.currentTimeMillis() - startTime));
//		System.out.println("RECORDS " + num);
		
		// release the memory occupied by the buffers
		sorter.dispose();
		this.memoryManager.release(memory);
	}
 

/**
 * The swap test fills the sort buffer and swaps all elements such that they are
 * backwards. It then resets the generator, goes backwards through the buffer
 * and compares for equality.
 */
@Test
public void testSwap() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	// swap the records
	int start = 0, end = num - 1;
	while (start < end) {
		sorter.swap(start++, end--);
	}
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = num - 1;
	while (i >= 0) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i--);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
public void testReset() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the buffer full with the first set of records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	sorter.reset();
	
	// write a second sequence of records. since the values are of fixed length, we must be able to write an equal number
	generator.reset();
	
	// write the buffer full with the first set of records
	int num2 = -1;
	do {
		generator.next(record);
		num2++;
	}
	while (sorter.write(record) && num2 < 3354624);
	
	Assert.assertEquals("The number of records written after the reset was not the same as before.", num, num2);
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = 0;
	while (i < num) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i++);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
public void testWriteAndIterator() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record));
	
	// re-read the records
	generator.reset();
	
	MutableObjectIterator<IntPair> iter = sorter.getIterator();
	IntPair readTarget = new IntPair();
	int count = 0;
	
	while ((readTarget = iter.next(readTarget)) != null) {
		count++;
		
		generator.next(record);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	Assert.assertEquals("Incorrect number of records", num, count);
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
	public void testWriteAndRead() throws Exception {
		final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
		final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
		
		FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
		RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
		
//		long startTime = System.currentTimeMillis();
		// write the records
		IntPair record = new IntPair();
		int num = -1;
		do {
			generator.next(record);
			num++;
		}
		while (sorter.write(record) && num < 3354624);
//		System.out.println("WRITE TIME " + (System.currentTimeMillis() - startTime));
		
		// re-read the records
		generator.reset();
		IntPair readTarget = new IntPair();
		
//		startTime = System.currentTimeMillis();
		int i = 0;
		while (i < num) {
			generator.next(record);
			readTarget = sorter.getRecord(readTarget, i++);
			
			int rk = readTarget.getKey();
			int gk = record.getKey();
			
			int rv = readTarget.getValue();
			int gv = record.getValue();
			
			if (gk != rk) {
				Assert.fail("The re-read key is wrong " + i);
			}
			if (gv != rv) {
				Assert.fail("The re-read value is wrong");
			}
		}
//		System.out.println("READ TIME " + (System.currentTimeMillis() - startTime));
//		System.out.println("RECORDS " + num);
		
		// release the memory occupied by the buffers
		sorter.dispose();
		this.memoryManager.release(memory);
	}
 

/**
 * The swap test fills the sort buffer and swaps all elements such that they are
 * backwards. It then resets the generator, goes backwards through the buffer
 * and compares for equality.
 */
@Test
public void testSwap() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	// swap the records
	int start = 0, end = num - 1;
	while (start < end) {
		sorter.swap(start++, end--);
	}
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = num - 1;
	while (i >= 0) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i--);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
public void testReset() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the buffer full with the first set of records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	sorter.reset();
	
	// write a second sequence of records. since the values are of fixed length, we must be able to write an equal number
	generator.reset();
	
	// write the buffer full with the first set of records
	int num2 = -1;
	do {
		generator.next(record);
		num2++;
	}
	while (sorter.write(record) && num2 < 3354624);
	
	Assert.assertEquals("The number of records written after the reset was not the same as before.", num, num2);
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = 0;
	while (i < num) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i++);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

/**
 * The swap test fills the sort buffer and swaps all elements such that they are
 * backwards. It then resets the generator, goes backwards through the buffer
 * and compares for equality.
 */
@Test
public void testSwap() throws Exception {
	final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
	final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
	
	FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
	RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
	
	// write the records
	IntPair record = new IntPair();
	int num = -1;
	do {
		generator.next(record);
		num++;
	}
	while (sorter.write(record) && num < 3354624);
	
	// swap the records
	int start = 0, end = num - 1;
	while (start < end) {
		sorter.swap(start++, end--);
	}
	
	// re-read the records
	generator.reset();
	IntPair readTarget = new IntPair();
	
	int i = num - 1;
	while (i >= 0) {
		generator.next(record);
		readTarget = sorter.getRecord(readTarget, i--);
		
		int rk = readTarget.getKey();
		int gk = record.getKey();
		
		int rv = readTarget.getValue();
		int gv = record.getValue();
		
		Assert.assertEquals("The re-read key is wrong", gk, rk);
		Assert.assertEquals("The re-read value is wrong", gv, rv);
	}
	
	// release the memory occupied by the buffers
	sorter.dispose();
	this.memoryManager.release(memory);
}
 

@Test
	public void testWriteAndRead() throws Exception {
		final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
		final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
		
		FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
		RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
		
//		long startTime = System.currentTimeMillis();
		// write the records
		IntPair record = new IntPair();
		int num = -1;
		do {
			generator.next(record);
			num++;
		}
		while (sorter.write(record) && num < 3354624);
//		System.out.println("WRITE TIME " + (System.currentTimeMillis() - startTime));
		
		// re-read the records
		generator.reset();
		IntPair readTarget = new IntPair();
		
//		startTime = System.currentTimeMillis();
		int i = 0;
		while (i < num) {
			generator.next(record);
			readTarget = sorter.getRecord(readTarget, i++);
			
			int rk = readTarget.getKey();
			int gk = record.getKey();
			
			int rv = readTarget.getValue();
			int gv = record.getValue();
			
			if (gk != rk) {
				Assert.fail("The re-read key is wrong " + i);
			}
			if (gv != rv) {
				Assert.fail("The re-read value is wrong");
			}
		}
//		System.out.println("READ TIME " + (System.currentTimeMillis() - startTime));
//		System.out.println("RECORDS " + num);
		
		// release the memory occupied by the buffers
		sorter.dispose();
		this.memoryManager.release(memory);
	}