org.apache.flink.shaded.guava18.com.google.common.collect.Ordering Java Examples

@Test
public void testWithIntPair() throws Exception {
	Random rnd = new Random(RANDOM_SEED);

	// varying the keyRange between 1000 and 1000000 can make a 5x speed difference
	// (because of cache misses (also in the segment arrays))
	final int keyRange = 1000000;
	final int valueRange = 10;
	final int numRecords = 1000000;

	final IntPairSerializer serializer = new IntPairSerializer();
	final TypeComparator<IntPair> comparator = new IntPairComparator();
	final ReduceFunction<IntPair> reducer = new SumReducer();

	// Create the InPlaceMutableHashTableWithJavaHashMap, which will provide the correct output.
	List<IntPair> expectedOutput = new ArrayList<>();
	InPlaceMutableHashTableWithJavaHashMap<IntPair, Integer> reference = new InPlaceMutableHashTableWithJavaHashMap<>(
		serializer, comparator, reducer, new CopyingListCollector<>(expectedOutput, serializer));

	// Create the InPlaceMutableHashTable to test
	final int numMemPages = keyRange * 32 / PAGE_SIZE; // memory use is proportional to the number of different keys
	List<IntPair> actualOutput = new ArrayList<>();

	InPlaceMutableHashTable<IntPair> table = new InPlaceMutableHashTable<>(
		serializer, comparator, getMemory(numMemPages, PAGE_SIZE));
	InPlaceMutableHashTable<IntPair>.ReduceFacade reduceFacade = table.new ReduceFacade(reducer,
		new CopyingListCollector<>(actualOutput, serializer), true);
	table.open();

	// Generate some input
	final List<IntPair> input = new ArrayList<>();
	for(int i = 0; i < numRecords; i++) {
		input.add(new IntPair(rnd.nextInt(keyRange), rnd.nextInt(valueRange)));
	}

	//System.out.println("start");
	//long start = System.currentTimeMillis();

	// Process the generated input
	final int numIntermingledEmits = 5;
	for (IntPair record: input) {
		reduceFacade.updateTableEntryWithReduce(serializer.copy(record));
		reference.updateTableEntryWithReduce(serializer.copy(record), record.getKey());
		if(rnd.nextDouble() < 1.0 / ((double)numRecords / numIntermingledEmits)) {
			// this will fire approx. numIntermingledEmits times
			reference.emitAndReset();
			reduceFacade.emitAndReset();
		}
	}
	reference.emitAndReset();
	reduceFacade.emit();
	table.close();

	//long end = System.currentTimeMillis();
	//System.out.println("stop, time: " + (end - start));

	// Check results

	assertEquals(expectedOutput.size(), actualOutput.size());

	Integer[] expectedValues = new Integer[expectedOutput.size()];
	for (int i = 0; i < expectedOutput.size(); i++) {
		expectedValues[i] = expectedOutput.get(i).getValue();
	}
	Integer[] actualValues = new Integer[actualOutput.size()];
	for (int i = 0; i < actualOutput.size(); i++) {
		actualValues[i] = actualOutput.get(i).getValue();
	}

	Arrays.sort(expectedValues, Ordering.<Integer>natural());
	Arrays.sort(actualValues, Ordering.<Integer>natural());
	assertArrayEquals(expectedValues, actualValues);
}
 

@Test
public void testWithLengthChangingReduceFunction() throws Exception {
	Random rnd = new Random(RANDOM_SEED);

	final int numKeys = 10000;
	final int numVals = 10;
	final int numRecords = numKeys * numVals;

	StringPairSerializer serializer = new StringPairSerializer();
	StringPairComparator comparator = new StringPairComparator();
	ReduceFunction<StringPair> reducer = new ConcatReducer();

	// Create the InPlaceMutableHashTableWithJavaHashMap, which will provide the correct output.
	List<StringPair> expectedOutput = new ArrayList<>();
	InPlaceMutableHashTableWithJavaHashMap<StringPair, String> reference = new InPlaceMutableHashTableWithJavaHashMap<>(
		serializer, comparator, reducer, new CopyingListCollector<>(expectedOutput, serializer));

	// Create the InPlaceMutableHashTable to test
	final int numMemPages = numRecords * 10 / PAGE_SIZE;

	List<StringPair> actualOutput = new ArrayList<>();

	InPlaceMutableHashTable<StringPair> table =
		new InPlaceMutableHashTable<>(serializer, comparator, getMemory(numMemPages, PAGE_SIZE));
	InPlaceMutableHashTable<StringPair>.ReduceFacade reduceFacade =
		table.new ReduceFacade(reducer, new CopyingListCollector<>(actualOutput, serializer), true);

	// The loop is for checking the feature that multiple open / close are possible.
	for(int j = 0; j < 3; j++) {
		table.open();

		// Test emit when table is empty
		reduceFacade.emit();

		// Process some manual stuff
		reference.updateTableEntryWithReduce(serializer.copy(new StringPair("foo", "bar")), "foo");
		reference.updateTableEntryWithReduce(serializer.copy(new StringPair("foo", "baz")), "foo");
		reference.updateTableEntryWithReduce(serializer.copy(new StringPair("alma", "xyz")), "alma");
		reduceFacade.updateTableEntryWithReduce(serializer.copy(new StringPair("foo", "bar")));
		reduceFacade.updateTableEntryWithReduce(serializer.copy(new StringPair("foo", "baz")));
		reduceFacade.updateTableEntryWithReduce(serializer.copy(new StringPair("alma", "xyz")));
		for (int i = 0; i < 5; i++) {
			reduceFacade.updateTableEntryWithReduce(serializer.copy(new StringPair("korte", "abc")));
			reference.updateTableEntryWithReduce(serializer.copy(new StringPair("korte", "abc")), "korte");
		}
		reference.emitAndReset();
		reduceFacade.emitAndReset();

		// Generate some input
		UniformStringPairGenerator gen = new UniformStringPairGenerator(numKeys, numVals, true);
		List<StringPair> input = new ArrayList<>();
		StringPair cur = new StringPair();
		while (gen.next(cur) != null) {
			input.add(serializer.copy(cur));
		}
		Collections.shuffle(input, rnd);

		// Process the generated input
		final int numIntermingledEmits = 5;
		for (StringPair record : input) {
			reference.updateTableEntryWithReduce(serializer.copy(record), record.getKey());
			reduceFacade.updateTableEntryWithReduce(serializer.copy(record));
			if (rnd.nextDouble() < 1.0 / ((double) numRecords / numIntermingledEmits)) {
				// this will fire approx. numIntermingledEmits times
				reference.emitAndReset();
				reduceFacade.emitAndReset();
			}
		}
		reference.emitAndReset();
		reduceFacade.emit();
		table.close();

		// Check results

		assertEquals(expectedOutput.size(), actualOutput.size());

		String[] expectedValues = new String[expectedOutput.size()];
		for (int i = 0; i < expectedOutput.size(); i++) {
			expectedValues[i] = expectedOutput.get(i).getValue();
		}
		String[] actualValues = new String[actualOutput.size()];
		for (int i = 0; i < actualOutput.size(); i++) {
			actualValues[i] = actualOutput.get(i).getValue();
		}

		Arrays.sort(expectedValues, Ordering.<String>natural());
		Arrays.sort(actualValues, Ordering.<String>natural());
		assertArrayEquals(expectedValues, actualValues);

		expectedOutput.clear();
		actualOutput.clear();
	}
}
 

@Test
public void testWithIntPair() throws Exception {
	Random rnd = new Random(RANDOM_SEED);

	// varying the keyRange between 1000 and 1000000 can make a 5x speed difference
	// (because of cache misses (also in the segment arrays))
	final int keyRange = 1000000;
	final int valueRange = 10;
	final int numRecords = 1000000;

	final IntPairSerializer serializer = new IntPairSerializer();
	final TypeComparator<IntPair> comparator = new IntPairComparator();
	final ReduceFunction<IntPair> reducer = new SumReducer();

	// Create the InPlaceMutableHashTableWithJavaHashMap, which will provide the correct output.
	List<IntPair> expectedOutput = new ArrayList<>();
	InPlaceMutableHashTableWithJavaHashMap<IntPair, Integer> reference = new InPlaceMutableHashTableWithJavaHashMap<>(
		serializer, comparator, reducer, new CopyingListCollector<>(expectedOutput, serializer));

	// Create the InPlaceMutableHashTable to test
	final int numMemPages = keyRange * 32 / PAGE_SIZE; // memory use is proportional to the number of different keys
	List<IntPair> actualOutput = new ArrayList<>();

	InPlaceMutableHashTable<IntPair> table = new InPlaceMutableHashTable<>(
		serializer, comparator, getMemory(numMemPages, PAGE_SIZE));
	InPlaceMutableHashTable<IntPair>.ReduceFacade reduceFacade = table.new ReduceFacade(reducer,
		new CopyingListCollector<>(actualOutput, serializer), true);
	table.open();

	// Generate some input
	final List<IntPair> input = new ArrayList<>();
	for(int i = 0; i < numRecords; i++) {
		input.add(new IntPair(rnd.nextInt(keyRange), rnd.nextInt(valueRange)));
	}

	//System.out.println("start");
	//long start = System.currentTimeMillis();

	// Process the generated input
	final int numIntermingledEmits = 5;
	for (IntPair record: input) {
		reduceFacade.updateTableEntryWithReduce(serializer.copy(record));
		reference.updateTableEntryWithReduce(serializer.copy(record), record.getKey());
		if(rnd.nextDouble() < 1.0 / ((double)numRecords / numIntermingledEmits)) {
			// this will fire approx. numIntermingledEmits times
			reference.emitAndReset();
			reduceFacade.emitAndReset();
		}
	}
	reference.emitAndReset();
	reduceFacade.emit();
	table.close();

	//long end = System.currentTimeMillis();
	//System.out.println("stop, time: " + (end - start));

	// Check results

	assertEquals(expectedOutput.size(), actualOutput.size());

	Integer[] expectedValues = new Integer[expectedOutput.size()];
	for (int i = 0; i < expectedOutput.size(); i++) {
		expectedValues[i] = expectedOutput.get(i).getValue();
	}
	Integer[] actualValues = new Integer[actualOutput.size()];
	for (int i = 0; i < actualOutput.size(); i++) {
		actualValues[i] = actualOutput.get(i).getValue();
	}

	Arrays.sort(expectedValues, Ordering.<Integer>natural());
	Arrays.sort(actualValues, Ordering.<Integer>natural());
	assertArrayEquals(expectedValues, actualValues);
}
 

@Test
public void testWithLengthChangingReduceFunction() throws Exception {
	Random rnd = new Random(RANDOM_SEED);

	final int numKeys = 10000;
	final int numVals = 10;
	final int numRecords = numKeys * numVals;

	StringPairSerializer serializer = new StringPairSerializer();
	StringPairComparator comparator = new StringPairComparator();
	ReduceFunction<StringPair> reducer = new ConcatReducer();

	// Create the InPlaceMutableHashTableWithJavaHashMap, which will provide the correct output.
	List<StringPair> expectedOutput = new ArrayList<>();
	InPlaceMutableHashTableWithJavaHashMap<StringPair, String> reference = new InPlaceMutableHashTableWithJavaHashMap<>(
		serializer, comparator, reducer, new CopyingListCollector<>(expectedOutput, serializer));

	// Create the InPlaceMutableHashTable to test
	final int numMemPages = numRecords * 10 / PAGE_SIZE;

	List<StringPair> actualOutput = new ArrayList<>();

	InPlaceMutableHashTable<StringPair> table =
		new InPlaceMutableHashTable<>(serializer, comparator, getMemory(numMemPages, PAGE_SIZE));
	InPlaceMutableHashTable<StringPair>.ReduceFacade reduceFacade =
		table.new ReduceFacade(reducer, new CopyingListCollector<>(actualOutput, serializer), true);

	// The loop is for checking the feature that multiple open / close are possible.
	for(int j = 0; j < 3; j++) {
		table.open();

		// Test emit when table is empty
		reduceFacade.emit();

		// Process some manual stuff
		reference.updateTableEntryWithReduce(serializer.copy(new StringPair("foo", "bar")), "foo");
		reference.updateTableEntryWithReduce(serializer.copy(new StringPair("foo", "baz")), "foo");
		reference.updateTableEntryWithReduce(serializer.copy(new StringPair("alma", "xyz")), "alma");
		reduceFacade.updateTableEntryWithReduce(serializer.copy(new StringPair("foo", "bar")));
		reduceFacade.updateTableEntryWithReduce(serializer.copy(new StringPair("foo", "baz")));
		reduceFacade.updateTableEntryWithReduce(serializer.copy(new StringPair("alma", "xyz")));
		for (int i = 0; i < 5; i++) {
			reduceFacade.updateTableEntryWithReduce(serializer.copy(new StringPair("korte", "abc")));
			reference.updateTableEntryWithReduce(serializer.copy(new StringPair("korte", "abc")), "korte");
		}
		reference.emitAndReset();
		reduceFacade.emitAndReset();

		// Generate some input
		UniformStringPairGenerator gen = new UniformStringPairGenerator(numKeys, numVals, true);
		List<StringPair> input = new ArrayList<>();
		StringPair cur = new StringPair();
		while (gen.next(cur) != null) {
			input.add(serializer.copy(cur));
		}
		Collections.shuffle(input, rnd);

		// Process the generated input
		final int numIntermingledEmits = 5;
		for (StringPair record : input) {
			reference.updateTableEntryWithReduce(serializer.copy(record), record.getKey());
			reduceFacade.updateTableEntryWithReduce(serializer.copy(record));
			if (rnd.nextDouble() < 1.0 / ((double) numRecords / numIntermingledEmits)) {
				// this will fire approx. numIntermingledEmits times
				reference.emitAndReset();
				reduceFacade.emitAndReset();
			}
		}
		reference.emitAndReset();
		reduceFacade.emit();
		table.close();

		// Check results

		assertEquals(expectedOutput.size(), actualOutput.size());

		String[] expectedValues = new String[expectedOutput.size()];
		for (int i = 0; i < expectedOutput.size(); i++) {
			expectedValues[i] = expectedOutput.get(i).getValue();
		}
		String[] actualValues = new String[actualOutput.size()];
		for (int i = 0; i < actualOutput.size(); i++) {
			actualValues[i] = actualOutput.get(i).getValue();
		}

		Arrays.sort(expectedValues, Ordering.<String>natural());
		Arrays.sort(actualValues, Ordering.<String>natural());
		assertArrayEquals(expectedValues, actualValues);

		expectedOutput.clear();
		actualOutput.clear();
	}
}
 

@Test
public void testWithIntPair() throws Exception {
	Random rnd = new Random(RANDOM_SEED);

	// varying the keyRange between 1000 and 1000000 can make a 5x speed difference
	// (because of cache misses (also in the segment arrays))
	final int keyRange = 1000000;
	final int valueRange = 10;
	final int numRecords = 1000000;

	final IntPairSerializer serializer = new IntPairSerializer();
	final TypeComparator<IntPair> comparator = new IntPairComparator();
	final ReduceFunction<IntPair> reducer = new SumReducer();

	// Create the InPlaceMutableHashTableWithJavaHashMap, which will provide the correct output.
	List<IntPair> expectedOutput = new ArrayList<>();
	InPlaceMutableHashTableWithJavaHashMap<IntPair, Integer> reference = new InPlaceMutableHashTableWithJavaHashMap<>(
		serializer, comparator, reducer, new CopyingListCollector<>(expectedOutput, serializer));

	// Create the InPlaceMutableHashTable to test
	final int numMemPages = keyRange * 32 / PAGE_SIZE; // memory use is proportional to the number of different keys
	List<IntPair> actualOutput = new ArrayList<>();

	InPlaceMutableHashTable<IntPair> table = new InPlaceMutableHashTable<>(
		serializer, comparator, getMemory(numMemPages, PAGE_SIZE));
	InPlaceMutableHashTable<IntPair>.ReduceFacade reduceFacade = table.new ReduceFacade(reducer,
		new CopyingListCollector<>(actualOutput, serializer), true);
	table.open();

	// Generate some input
	final List<IntPair> input = new ArrayList<>();
	for(int i = 0; i < numRecords; i++) {
		input.add(new IntPair(rnd.nextInt(keyRange), rnd.nextInt(valueRange)));
	}

	//System.out.println("start");
	//long start = System.currentTimeMillis();

	// Process the generated input
	final int numIntermingledEmits = 5;
	for (IntPair record: input) {
		reduceFacade.updateTableEntryWithReduce(serializer.copy(record));
		reference.updateTableEntryWithReduce(serializer.copy(record), record.getKey());
		if(rnd.nextDouble() < 1.0 / ((double)numRecords / numIntermingledEmits)) {
			// this will fire approx. numIntermingledEmits times
			reference.emitAndReset();
			reduceFacade.emitAndReset();
		}
	}
	reference.emitAndReset();
	reduceFacade.emit();
	table.close();

	//long end = System.currentTimeMillis();
	//System.out.println("stop, time: " + (end - start));

	// Check results

	assertEquals(expectedOutput.size(), actualOutput.size());

	Integer[] expectedValues = new Integer[expectedOutput.size()];
	for (int i = 0; i < expectedOutput.size(); i++) {
		expectedValues[i] = expectedOutput.get(i).getValue();
	}
	Integer[] actualValues = new Integer[actualOutput.size()];
	for (int i = 0; i < actualOutput.size(); i++) {
		actualValues[i] = actualOutput.get(i).getValue();
	}

	Arrays.sort(expectedValues, Ordering.<Integer>natural());
	Arrays.sort(actualValues, Ordering.<Integer>natural());
	assertArrayEquals(expectedValues, actualValues);
}
 

@Test
public void testWithLengthChangingReduceFunction() throws Exception {
	Random rnd = new Random(RANDOM_SEED);

	final int numKeys = 10000;
	final int numVals = 10;
	final int numRecords = numKeys * numVals;

	StringPairSerializer serializer = new StringPairSerializer();
	StringPairComparator comparator = new StringPairComparator();
	ReduceFunction<StringPair> reducer = new ConcatReducer();

	// Create the InPlaceMutableHashTableWithJavaHashMap, which will provide the correct output.
	List<StringPair> expectedOutput = new ArrayList<>();
	InPlaceMutableHashTableWithJavaHashMap<StringPair, String> reference = new InPlaceMutableHashTableWithJavaHashMap<>(
		serializer, comparator, reducer, new CopyingListCollector<>(expectedOutput, serializer));

	// Create the InPlaceMutableHashTable to test
	final int numMemPages = numRecords * 10 / PAGE_SIZE;

	List<StringPair> actualOutput = new ArrayList<>();

	InPlaceMutableHashTable<StringPair> table =
		new InPlaceMutableHashTable<>(serializer, comparator, getMemory(numMemPages, PAGE_SIZE));
	InPlaceMutableHashTable<StringPair>.ReduceFacade reduceFacade =
		table.new ReduceFacade(reducer, new CopyingListCollector<>(actualOutput, serializer), true);

	// The loop is for checking the feature that multiple open / close are possible.
	for(int j = 0; j < 3; j++) {
		table.open();

		// Test emit when table is empty
		reduceFacade.emit();

		// Process some manual stuff
		reference.updateTableEntryWithReduce(serializer.copy(new StringPair("foo", "bar")), "foo");
		reference.updateTableEntryWithReduce(serializer.copy(new StringPair("foo", "baz")), "foo");
		reference.updateTableEntryWithReduce(serializer.copy(new StringPair("alma", "xyz")), "alma");
		reduceFacade.updateTableEntryWithReduce(serializer.copy(new StringPair("foo", "bar")));
		reduceFacade.updateTableEntryWithReduce(serializer.copy(new StringPair("foo", "baz")));
		reduceFacade.updateTableEntryWithReduce(serializer.copy(new StringPair("alma", "xyz")));
		for (int i = 0; i < 5; i++) {
			reduceFacade.updateTableEntryWithReduce(serializer.copy(new StringPair("korte", "abc")));
			reference.updateTableEntryWithReduce(serializer.copy(new StringPair("korte", "abc")), "korte");
		}
		reference.emitAndReset();
		reduceFacade.emitAndReset();

		// Generate some input
		UniformStringPairGenerator gen = new UniformStringPairGenerator(numKeys, numVals, true);
		List<StringPair> input = new ArrayList<>();
		StringPair cur = new StringPair();
		while (gen.next(cur) != null) {
			input.add(serializer.copy(cur));
		}
		Collections.shuffle(input, rnd);

		// Process the generated input
		final int numIntermingledEmits = 5;
		for (StringPair record : input) {
			reference.updateTableEntryWithReduce(serializer.copy(record), record.getKey());
			reduceFacade.updateTableEntryWithReduce(serializer.copy(record));
			if (rnd.nextDouble() < 1.0 / ((double) numRecords / numIntermingledEmits)) {
				// this will fire approx. numIntermingledEmits times
				reference.emitAndReset();
				reduceFacade.emitAndReset();
			}
		}
		reference.emitAndReset();
		reduceFacade.emit();
		table.close();

		// Check results

		assertEquals(expectedOutput.size(), actualOutput.size());

		String[] expectedValues = new String[expectedOutput.size()];
		for (int i = 0; i < expectedOutput.size(); i++) {
			expectedValues[i] = expectedOutput.get(i).getValue();
		}
		String[] actualValues = new String[actualOutput.size()];
		for (int i = 0; i < actualOutput.size(); i++) {
			actualValues[i] = actualOutput.get(i).getValue();
		}

		Arrays.sort(expectedValues, Ordering.<String>natural());
		Arrays.sort(actualValues, Ordering.<String>natural());
		assertArrayEquals(expectedValues, actualValues);

		expectedOutput.clear();
		actualOutput.clear();
	}
}