org.apache.flink.runtime.operators.sort.CombiningUnilateralSortMerger Java Examples

private void initInputLocalStrategy(int inputNum) throws Exception {
	// check if there is already a strategy
	if (this.localStrategies[inputNum] != null) {
		throw new IllegalStateException();
	}

	// now set up the local strategy
	final LocalStrategy localStrategy = this.config.getInputLocalStrategy(inputNum);
	if (localStrategy != null) {
		switch (localStrategy) {
		case NONE:
			// the input is as it is
			this.inputs[inputNum] = this.inputIterators[inputNum];
			break;
		case SORT:
			@SuppressWarnings({ "rawtypes", "unchecked" })
			UnilateralSortMerger<?> sorter = new UnilateralSortMerger(getMemoryManager(), getIOManager(),
				this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.config.getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = sorter;
			break;
		case COMBININGSORT:
			// sanity check this special case!
			// this still breaks a bit of the abstraction!
			// we should have nested configurations for the local strategies to solve that
			if (inputNum != 0) {
				throw new IllegalStateException("Performing combining sort outside a (group)reduce task!");
			}

			// instantiate ourselves a combiner. we should not use the stub, because the sort and the
			// subsequent (group)reduce would otherwise share it multi-threaded
			final Class<S> userCodeFunctionType = this.driver.getStubType();
			if (userCodeFunctionType == null) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}
			final S localStub;
			try {
				localStub = initStub(userCodeFunctionType);
			} catch (Exception e) {
				throw new RuntimeException("Initializing the user code and the configuration failed" +
						(e.getMessage() == null ? "." : ": " + e.getMessage()), e);
			}
			
			if (!(localStub instanceof GroupCombineFunction)) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}

			@SuppressWarnings({ "rawtypes", "unchecked" })
			CombiningUnilateralSortMerger<?> cSorter = new CombiningUnilateralSortMerger(
				(GroupCombineFunction) localStub, getMemoryManager(), getIOManager(), this.inputIterators[inputNum],
				this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.getTaskConfig().getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			cSorter.setUdfConfiguration(this.config.getStubParameters());

			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = cSorter;
			break;
		default:
			throw new Exception("Unrecognized local strategy provided: " + localStrategy.name());
		}
	} else {
		// no local strategy in the config
		this.inputs[inputNum] = this.inputIterators[inputNum];
	}
}
 

@Test
public void testSingleLevelMergeCombiningReduceTask()
{
	final int keyCnt = 8192;
	final int valCnt = 8;
	
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	CombiningUnilateralSortMerger<Record> sorter = null;
	try {
		sorter = new CombiningUnilateralSortMerger<>(new MockCombiningReduceStub(),
			getMemoryManager(), getIOManager(), new UniformRecordGenerator(keyCnt, valCnt, false), 
			getContainingTask(), RecordSerializerFactory.get(), this.comparator.duplicate(),
				this.perSortFractionMem,
				2, 0.8f, true /* use large record handler */, true);
		addInput(sorter.getIterator());
		
		GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
		testDriver(testTask, MockCombiningReduceStub.class);
	} catch (Exception e) {
		LOG.info("Exception while running the test task.", e);
		Assert.fail("Invoke method caused exception: " + e.getMessage());
	} finally {
		if (sorter != null) {
			sorter.close();
		}
	}
	
	int expSum = 0;
	for (int i = 1; i < valCnt; i++) {
		expSum += i;
	}
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+keyCnt, this.outList.size() == keyCnt);
	
	for (Record record : this.outList) {
		Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == expSum-record.getField(0, IntValue.class).getValue());
	}
	
	this.outList.clear();
}
 

@Test
public void testMultiLevelMergeCombiningReduceTask() {

	int keyCnt = 32768;
	int valCnt = 8;
	
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	CombiningUnilateralSortMerger<Record> sorter = null;
	try {
		sorter = new CombiningUnilateralSortMerger<>(new MockCombiningReduceStub(),
			getMemoryManager(), getIOManager(), new UniformRecordGenerator(keyCnt, valCnt, false), 
			getContainingTask(), RecordSerializerFactory.get(), this.comparator.duplicate(),
				this.perSortFractionMem,
				2, 0.8f, true /* use large record handler */, false);
		addInput(sorter.getIterator());
		
		GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
		testDriver(testTask, MockCombiningReduceStub.class);
	} catch (Exception e) {
		LOG.info("Exception while running the test task.", e);
		Assert.fail("Invoke method caused exception: " + e.getMessage());
	} finally {
		if (sorter != null) {
			sorter.close();
		}
	}
	
	int expSum = 0;
	for (int i = 1; i < valCnt; i++) {
		expSum += i;
	}
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+keyCnt, this.outList.size() == keyCnt);
	
	for (Record record : this.outList) {
		Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == expSum-record.getField(0, IntValue.class).getValue());
	}
	
	this.outList.clear();
	
}
 

@Test
public void testCombiningReduceTask() {
	final int keyCnt = 100;
	final int valCnt = 20;
	
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	CombiningUnilateralSortMerger<Record> sorter = null;
	try {
		sorter = new CombiningUnilateralSortMerger<>(new MockCombiningReduceStub(),
			getMemoryManager(), getIOManager(), new UniformRecordGenerator(keyCnt, valCnt, false), 
			getContainingTask(), RecordSerializerFactory.get(), this.comparator.duplicate(), this.perSortFractionMem,
				4, 0.8f, true /* use large record handler */, true);
		addInput(sorter.getIterator());
		
		GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
		testDriver(testTask, MockCombiningReduceStub.class);
	} catch (Exception e) {
		LOG.info("Exception while running the test task.", e);
		Assert.fail("Invoke method caused exception: " + e.getMessage());
	} finally {
		if (sorter != null) {
			sorter.close();
		}
	}
	
	int expSum = 0;
	for (int i = 1; i < valCnt; i++) {
		expSum += i;
	}
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+keyCnt, this.outList.size() == keyCnt);
	
	for(Record record : this.outList) {
		Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == expSum-record.getField(0, IntValue.class).getValue());
	}
	
	this.outList.clear();
	
}
 

private void initInputLocalStrategy(int inputNum) throws Exception {
	// check if there is already a strategy
	if (this.localStrategies[inputNum] != null) {
		throw new IllegalStateException();
	}

	// now set up the local strategy
	final LocalStrategy localStrategy = this.config.getInputLocalStrategy(inputNum);
	if (localStrategy != null) {
		switch (localStrategy) {
		case NONE:
			// the input is as it is
			this.inputs[inputNum] = this.inputIterators[inputNum];
			break;
		case SORT:
			@SuppressWarnings({ "rawtypes", "unchecked" })
			UnilateralSortMerger<?> sorter = new UnilateralSortMerger(getMemoryManager(), getIOManager(),
				this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.config.getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = sorter;
			break;
		case COMBININGSORT:
			// sanity check this special case!
			// this still breaks a bit of the abstraction!
			// we should have nested configurations for the local strategies to solve that
			if (inputNum != 0) {
				throw new IllegalStateException("Performing combining sort outside a (group)reduce task!");
			}

			// instantiate ourselves a combiner. we should not use the stub, because the sort and the
			// subsequent (group)reduce would otherwise share it multi-threaded
			final Class<S> userCodeFunctionType = this.driver.getStubType();
			if (userCodeFunctionType == null) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}
			final S localStub;
			try {
				localStub = initStub(userCodeFunctionType);
			} catch (Exception e) {
				throw new RuntimeException("Initializing the user code and the configuration failed" +
						(e.getMessage() == null ? "." : ": " + e.getMessage()), e);
			}
			
			if (!(localStub instanceof GroupCombineFunction)) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}

			@SuppressWarnings({ "rawtypes", "unchecked" })
			CombiningUnilateralSortMerger<?> cSorter = new CombiningUnilateralSortMerger(
				(GroupCombineFunction) localStub, getMemoryManager(), getIOManager(), this.inputIterators[inputNum],
				this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.getTaskConfig().getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			cSorter.setUdfConfiguration(this.config.getStubParameters());

			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = cSorter;
			break;
		default:
			throw new Exception("Unrecognized local strategy provided: " + localStrategy.name());
		}
	} else {
		// no local strategy in the config
		this.inputs[inputNum] = this.inputIterators[inputNum];
	}
}
 

@Test
public void testSingleLevelMergeCombiningReduceTask()
{
	final int keyCnt = 8192;
	final int valCnt = 8;
	
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	CombiningUnilateralSortMerger<Record> sorter = null;
	try {
		sorter = new CombiningUnilateralSortMerger<>(new MockCombiningReduceStub(),
			getMemoryManager(), getIOManager(), new UniformRecordGenerator(keyCnt, valCnt, false), 
			getContainingTask(), RecordSerializerFactory.get(), this.comparator.duplicate(),
				this.perSortFractionMem,
				2, 0.8f, true /* use large record handler */, true);
		addInput(sorter.getIterator());
		
		GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
		testDriver(testTask, MockCombiningReduceStub.class);
	} catch (Exception e) {
		LOG.info("Exception while running the test task.", e);
		Assert.fail("Invoke method caused exception: " + e.getMessage());
	} finally {
		if (sorter != null) {
			sorter.close();
		}
	}
	
	int expSum = 0;
	for (int i = 1; i < valCnt; i++) {
		expSum += i;
	}
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+keyCnt, this.outList.size() == keyCnt);
	
	for (Record record : this.outList) {
		Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == expSum-record.getField(0, IntValue.class).getValue());
	}
	
	this.outList.clear();
}
 

@Test
public void testMultiLevelMergeCombiningReduceTask() {

	int keyCnt = 32768;
	int valCnt = 8;
	
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	CombiningUnilateralSortMerger<Record> sorter = null;
	try {
		sorter = new CombiningUnilateralSortMerger<>(new MockCombiningReduceStub(),
			getMemoryManager(), getIOManager(), new UniformRecordGenerator(keyCnt, valCnt, false), 
			getContainingTask(), RecordSerializerFactory.get(), this.comparator.duplicate(),
				this.perSortFractionMem,
				2, 0.8f, true /* use large record handler */, false);
		addInput(sorter.getIterator());
		
		GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
		testDriver(testTask, MockCombiningReduceStub.class);
	} catch (Exception e) {
		LOG.info("Exception while running the test task.", e);
		Assert.fail("Invoke method caused exception: " + e.getMessage());
	} finally {
		if (sorter != null) {
			sorter.close();
		}
	}
	
	int expSum = 0;
	for (int i = 1; i < valCnt; i++) {
		expSum += i;
	}
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+keyCnt, this.outList.size() == keyCnt);
	
	for (Record record : this.outList) {
		Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == expSum-record.getField(0, IntValue.class).getValue());
	}
	
	this.outList.clear();
	
}
 

@Test
public void testCombiningReduceTask() {
	final int keyCnt = 100;
	final int valCnt = 20;
	
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	CombiningUnilateralSortMerger<Record> sorter = null;
	try {
		sorter = new CombiningUnilateralSortMerger<>(new MockCombiningReduceStub(),
			getMemoryManager(), getIOManager(), new UniformRecordGenerator(keyCnt, valCnt, false), 
			getContainingTask(), RecordSerializerFactory.get(), this.comparator.duplicate(), this.perSortFractionMem,
				4, 0.8f, true /* use large record handler */, true);
		addInput(sorter.getIterator());
		
		GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
		testDriver(testTask, MockCombiningReduceStub.class);
	} catch (Exception e) {
		LOG.info("Exception while running the test task.", e);
		Assert.fail("Invoke method caused exception: " + e.getMessage());
	} finally {
		if (sorter != null) {
			sorter.close();
		}
	}
	
	int expSum = 0;
	for (int i = 1; i < valCnt; i++) {
		expSum += i;
	}
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+keyCnt, this.outList.size() == keyCnt);
	
	for(Record record : this.outList) {
		Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == expSum-record.getField(0, IntValue.class).getValue());
	}
	
	this.outList.clear();
	
}
 

private void initInputLocalStrategy(int inputNum) throws Exception {
	// check if there is already a strategy
	if (this.localStrategies[inputNum] != null) {
		throw new IllegalStateException();
	}

	// now set up the local strategy
	final LocalStrategy localStrategy = this.config.getInputLocalStrategy(inputNum);
	if (localStrategy != null) {
		switch (localStrategy) {
		case NONE:
			// the input is as it is
			this.inputs[inputNum] = this.inputIterators[inputNum];
			break;
		case SORT:
			@SuppressWarnings({ "rawtypes", "unchecked" })
			UnilateralSortMerger<?> sorter = new UnilateralSortMerger(getMemoryManager(), getIOManager(),
				this.inputIterators[inputNum], this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.config.getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = sorter;
			break;
		case COMBININGSORT:
			// sanity check this special case!
			// this still breaks a bit of the abstraction!
			// we should have nested configurations for the local strategies to solve that
			if (inputNum != 0) {
				throw new IllegalStateException("Performing combining sort outside a (group)reduce task!");
			}

			// instantiate ourselves a combiner. we should not use the stub, because the sort and the
			// subsequent (group)reduce would otherwise share it multi-threaded
			final Class<S> userCodeFunctionType = this.driver.getStubType();
			if (userCodeFunctionType == null) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}
			final S localStub;
			try {
				localStub = initStub(userCodeFunctionType);
			} catch (Exception e) {
				throw new RuntimeException("Initializing the user code and the configuration failed" +
						(e.getMessage() == null ? "." : ": " + e.getMessage()), e);
			}
			
			if (!(localStub instanceof GroupCombineFunction)) {
				throw new IllegalStateException("Performing combining sort outside a reduce task!");
			}

			@SuppressWarnings({ "rawtypes", "unchecked" })
			CombiningUnilateralSortMerger<?> cSorter = new CombiningUnilateralSortMerger(
				(GroupCombineFunction) localStub, getMemoryManager(), getIOManager(), this.inputIterators[inputNum],
				this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
				this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
				this.config.getSpillingThresholdInput(inputNum), this.getTaskConfig().getUseLargeRecordHandler(),
				this.getExecutionConfig().isObjectReuseEnabled());
			cSorter.setUdfConfiguration(this.config.getStubParameters());

			// set the input to null such that it will be lazily fetched from the input strategy
			this.inputs[inputNum] = null;
			this.localStrategies[inputNum] = cSorter;
			break;
		default:
			throw new Exception("Unrecognized local strategy provided: " + localStrategy.name());
		}
	} else {
		// no local strategy in the config
		this.inputs[inputNum] = this.inputIterators[inputNum];
	}
}
 

@Test
public void testSingleLevelMergeCombiningReduceTask()
{
	final int keyCnt = 8192;
	final int valCnt = 8;
	
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	CombiningUnilateralSortMerger<Record> sorter = null;
	try {
		sorter = new CombiningUnilateralSortMerger<>(new MockCombiningReduceStub(),
			getMemoryManager(), getIOManager(), new UniformRecordGenerator(keyCnt, valCnt, false), 
			getContainingTask(), RecordSerializerFactory.get(), this.comparator.duplicate(),
				this.perSortFractionMem,
				2, 0.8f, true /* use large record handler */, true);
		addInput(sorter.getIterator());
		
		GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
		testDriver(testTask, MockCombiningReduceStub.class);
	} catch (Exception e) {
		LOG.info("Exception while running the test task.", e);
		Assert.fail("Invoke method caused exception: " + e.getMessage());
	} finally {
		if (sorter != null) {
			sorter.close();
		}
	}
	
	int expSum = 0;
	for (int i = 1; i < valCnt; i++) {
		expSum += i;
	}
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+keyCnt, this.outList.size() == keyCnt);
	
	for (Record record : this.outList) {
		Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == expSum-record.getField(0, IntValue.class).getValue());
	}
	
	this.outList.clear();
}
 

@Test
public void testMultiLevelMergeCombiningReduceTask() {

	int keyCnt = 32768;
	int valCnt = 8;
	
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	CombiningUnilateralSortMerger<Record> sorter = null;
	try {
		sorter = new CombiningUnilateralSortMerger<>(new MockCombiningReduceStub(),
			getMemoryManager(), getIOManager(), new UniformRecordGenerator(keyCnt, valCnt, false), 
			getContainingTask(), RecordSerializerFactory.get(), this.comparator.duplicate(),
				this.perSortFractionMem,
				2, 0.8f, true /* use large record handler */, false);
		addInput(sorter.getIterator());
		
		GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
		testDriver(testTask, MockCombiningReduceStub.class);
	} catch (Exception e) {
		LOG.info("Exception while running the test task.", e);
		Assert.fail("Invoke method caused exception: " + e.getMessage());
	} finally {
		if (sorter != null) {
			sorter.close();
		}
	}
	
	int expSum = 0;
	for (int i = 1; i < valCnt; i++) {
		expSum += i;
	}
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+keyCnt, this.outList.size() == keyCnt);
	
	for (Record record : this.outList) {
		Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == expSum-record.getField(0, IntValue.class).getValue());
	}
	
	this.outList.clear();
	
}
 

@Test
public void testCombiningReduceTask() {
	final int keyCnt = 100;
	final int valCnt = 20;
	
	addDriverComparator(this.comparator);
	setOutput(this.outList);
	getTaskConfig().setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
	
	CombiningUnilateralSortMerger<Record> sorter = null;
	try {
		sorter = new CombiningUnilateralSortMerger<>(new MockCombiningReduceStub(),
			getMemoryManager(), getIOManager(), new UniformRecordGenerator(keyCnt, valCnt, false), 
			getContainingTask(), RecordSerializerFactory.get(), this.comparator.duplicate(), this.perSortFractionMem,
				4, 0.8f, true /* use large record handler */, true);
		addInput(sorter.getIterator());
		
		GroupReduceDriver<Record, Record> testTask = new GroupReduceDriver<>();
	
		testDriver(testTask, MockCombiningReduceStub.class);
	} catch (Exception e) {
		LOG.info("Exception while running the test task.", e);
		Assert.fail("Invoke method caused exception: " + e.getMessage());
	} finally {
		if (sorter != null) {
			sorter.close();
		}
	}
	
	int expSum = 0;
	for (int i = 1; i < valCnt; i++) {
		expSum += i;
	}
	
	Assert.assertTrue("Resultset size was "+this.outList.size()+". Expected was "+keyCnt, this.outList.size() == keyCnt);
	
	for(Record record : this.outList) {
		Assert.assertTrue("Incorrect result", record.getField(1, IntValue.class).getValue() == expSum-record.getField(0, IntValue.class).getValue());
	}
	
	this.outList.clear();
	
}