org.apache.flink.runtime.io.network.api.writer.ChannelSelector Java Examples

@Test
public void testPartitionHash() {
	// Test for IntValue
	verifyPartitionHashSelectedChannels(50000, 100, RecordType.INTEGER);
	// Test for StringValue
	verifyPartitionHashSelectedChannels(10000, 100, RecordType.STRING);

	// Test hash corner cases
	final TestIntComparator testIntComp = new TestIntComparator();
	final ChannelSelector<SerializationDelegate<Integer>> selector = createChannelSelector(
		ShipStrategyType.PARTITION_HASH, testIntComp, 100);
	final SerializationDelegate<Integer> serializationDelegate = new SerializationDelegate<>(new IntSerializer());

	assertPartitionHashSelectedChannels(selector, serializationDelegate, Integer.MIN_VALUE, 100);
	assertPartitionHashSelectedChannels(selector, serializationDelegate, -1, 100);
	assertPartitionHashSelectedChannels(selector, serializationDelegate, 0, 100);
	assertPartitionHashSelectedChannels(selector, serializationDelegate, 1, 100);
	assertPartitionHashSelectedChannels(selector, serializationDelegate, Integer.MAX_VALUE, 100);
}
 

private boolean verifyWrongPartitionHashKey(int position, int fieldNum) {
	final TypeComparator<Record> comparator = new RecordComparatorFactory(
		new int[] {position}, new Class[] {IntValue.class}).createComparator();
	final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(
		ShipStrategyType.PARTITION_HASH, comparator, 100);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());

	Record record = new Record(2);
	record.setField(fieldNum, new IntValue(1));
	delegate.setInstance(record);

	try {
		selector.selectChannel(delegate);
	} catch (NullKeyFieldException re) {
		Assert.assertEquals(position, re.getFieldNumber());
		return true;
	}
	return false;
}
 

@Test
public void testPartitionHash() {
	// Test for IntValue
	verifyPartitionHashSelectedChannels(50000, 100, RecordType.INTEGER);
	// Test for StringValue
	verifyPartitionHashSelectedChannels(10000, 100, RecordType.STRING);

	// Test hash corner cases
	final TestIntComparator testIntComp = new TestIntComparator();
	final ChannelSelector<SerializationDelegate<Integer>> selector = createChannelSelector(
		ShipStrategyType.PARTITION_HASH, testIntComp, 100);
	final SerializationDelegate<Integer> serializationDelegate = new SerializationDelegate<>(new IntSerializer());

	assertPartitionHashSelectedChannels(selector, serializationDelegate, Integer.MIN_VALUE, 100);
	assertPartitionHashSelectedChannels(selector, serializationDelegate, -1, 100);
	assertPartitionHashSelectedChannels(selector, serializationDelegate, 0, 100);
	assertPartitionHashSelectedChannels(selector, serializationDelegate, 1, 100);
	assertPartitionHashSelectedChannels(selector, serializationDelegate, Integer.MAX_VALUE, 100);
}
 

private boolean verifyWrongPartitionHashKey(int position, int fieldNum) {
	final TypeComparator<Record> comparator = new RecordComparatorFactory(
		new int[] {position}, new Class[] {IntValue.class}).createComparator();
	final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(
		ShipStrategyType.PARTITION_HASH, comparator, 100);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());

	Record record = new Record(2);
	record.setField(fieldNum, new IntValue(1));
	delegate.setInstance(record);

	try {
		selector.selectChannel(delegate);
	} catch (NullKeyFieldException re) {
		Assert.assertEquals(position, re.getFieldNumber());
		return true;
	}
	return false;
}
 

@Test
public void testPartitionHash() {
	// Test for IntValue
	verifyPartitionHashSelectedChannels(50000, 100, RecordType.INTEGER);
	// Test for StringValue
	verifyPartitionHashSelectedChannels(10000, 100, RecordType.STRING);

	// Test hash corner cases
	final TestIntComparator testIntComp = new TestIntComparator();
	final ChannelSelector<SerializationDelegate<Integer>> selector = createChannelSelector(
		ShipStrategyType.PARTITION_HASH, testIntComp, 100);
	final SerializationDelegate<Integer> serializationDelegate = new SerializationDelegate<>(new IntSerializer());

	assertPartitionHashSelectedChannels(selector, serializationDelegate, Integer.MIN_VALUE, 100);
	assertPartitionHashSelectedChannels(selector, serializationDelegate, -1, 100);
	assertPartitionHashSelectedChannels(selector, serializationDelegate, 0, 100);
	assertPartitionHashSelectedChannels(selector, serializationDelegate, 1, 100);
	assertPartitionHashSelectedChannels(selector, serializationDelegate, Integer.MAX_VALUE, 100);
}
 

private boolean verifyWrongPartitionHashKey(int position, int fieldNum) {
	final TypeComparator<Record> comparator = new RecordComparatorFactory(
		new int[] {position}, new Class[] {IntValue.class}).createComparator();
	final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(
		ShipStrategyType.PARTITION_HASH, comparator, 100);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());

	Record record = new Record(2);
	record.setField(fieldNum, new IntValue(1));
	delegate.setInstance(record);

	try {
		selector.selectChannel(delegate);
	} catch (NullKeyFieldException re) {
		Assert.assertEquals(position, re.getFieldNumber());
		return true;
	}
	return false;
}
 

private void assertPartitionHashSelectedChannels(
		ChannelSelector selector,
		SerializationDelegate<Integer> serializationDelegate,
		int record,
		int numberOfChannels) {
	serializationDelegate.setInstance(record);
	int selectedChannel = selector.selectChannel(serializationDelegate);

	assertTrue(selectedChannel >= 0 && selectedChannel <= numberOfChannels - 1);
}
 

private void assertSelectedChannel(
	ChannelSelector<StringValue> selector,
	StringValue record,
	int expectedChannel) {

	int actualResult = selector.selectChannel(record);
	assertEquals(expectedChannel, actualResult);
}
 

private void assertPartitionHashSelectedChannels(
		ChannelSelector selector,
		SerializationDelegate<Integer> serializationDelegate,
		int record,
		int numberOfChannels) {
	serializationDelegate.setInstance(record);
	int selectedChannel = selector.selectChannel(serializationDelegate);

	assertTrue(selectedChannel >= 0 && selectedChannel <= numberOfChannels - 1);
}
 

private ChannelSelector createChannelSelector(
		ShipStrategyType shipStrategyType,
		TypeComparator comparator,
		int numberOfChannels) {
	final ChannelSelector selector = new OutputEmitter<>(shipStrategyType, comparator);
	selector.setup(numberOfChannels);
	assertEquals(shipStrategyType == ShipStrategyType.BROADCAST, selector.isBroadcast());
	return selector;
}
 

private int[] getSelectedChannelsHitCount(
		ShipStrategyType shipStrategyType,
		int numRecords,
		int numberOfChannels,
		Enum recordType) {
	final TypeComparator<Record> comparator = new RecordComparatorFactory(
		new int[] {0}, new Class[] {recordType == RecordType.INTEGER ? IntValue.class : StringValue.class}).createComparator();
	final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(shipStrategyType, comparator, numberOfChannels);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());

	return getSelectedChannelsHitCount(selector, delegate, recordType, numRecords, numberOfChannels);
}
 

@Test
public void testMultiKeys() {
	final int numberOfChannels = 100;
	final int numRecords = 5000;
	final TypeComparator<Record> multiComp = new RecordComparatorFactory(
		new int[] {0,1, 3}, new Class[] {IntValue.class, StringValue.class, DoubleValue.class}).createComparator();

	final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(
		ShipStrategyType.PARTITION_HASH, multiComp, numberOfChannels);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());

	int[] hits = new int[numberOfChannels];
	for (int i = 0; i < numRecords; i++) {
		Record record = new Record(4);
		record.setField(0, new IntValue(i));
		record.setField(1, new StringValue("AB" + i + "CD" + i));
		record.setField(3, new DoubleValue(i * 3.141d));
		delegate.setInstance(record);

		int channel = selector.selectChannel(delegate);
		hits[channel]++;
	}

	int totalHitCount = 0;
	for (int hit : hits) {
		assertTrue(hit > 0);
		totalHitCount += hit;
	}
	assertTrue(totalHitCount == numRecords);
}
 

private void assertSelectedChannel(
	ChannelSelector<StringValue> selector,
	StringValue record,
	int expectedChannel) {

	int actualResult = selector.selectChannel(record);
	assertEquals(expectedChannel, actualResult);
}
 

private int[] getSelectedChannelsHitCount(
		ShipStrategyType shipStrategyType,
		int numRecords,
		int numberOfChannels,
		Enum recordType) {
	final TypeComparator<Record> comparator = new RecordComparatorFactory(
		new int[] {0}, new Class[] {recordType == RecordType.INTEGER ? IntValue.class : StringValue.class}).createComparator();
	final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(shipStrategyType, comparator, numberOfChannels);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());

	return getSelectedChannelsHitCount(selector, delegate, recordType, numRecords, numberOfChannels);
}
 

private ChannelSelector createChannelSelector(
		ShipStrategyType shipStrategyType,
		TypeComparator comparator,
		int numberOfChannels) {
	final ChannelSelector selector = new OutputEmitter<>(shipStrategyType, comparator);
	selector.setup(numberOfChannels);
	assertEquals(shipStrategyType == ShipStrategyType.BROADCAST, selector.isBroadcast());
	return selector;
}
 

private int[] getSelectedChannelsHitCount(
		ShipStrategyType shipStrategyType,
		int numRecords,
		int numberOfChannels,
		Enum recordType) {
	final TypeComparator<Record> comparator = new RecordComparatorFactory(
		new int[] {0}, new Class[] {recordType == RecordType.INTEGER ? IntValue.class : StringValue.class}).createComparator();
	final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(shipStrategyType, comparator, numberOfChannels);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());

	return getSelectedChannelsHitCount(selector, delegate, recordType, numRecords, numberOfChannels);
}
 

@Test
public void testMultiKeys() {
	final int numberOfChannels = 100;
	final int numRecords = 5000;
	final TypeComparator<Record> multiComp = new RecordComparatorFactory(
		new int[] {0,1, 3}, new Class[] {IntValue.class, StringValue.class, DoubleValue.class}).createComparator();

	final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(
		ShipStrategyType.PARTITION_HASH, multiComp, numberOfChannels);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());

	int[] hits = new int[numberOfChannels];
	for (int i = 0; i < numRecords; i++) {
		Record record = new Record(4);
		record.setField(0, new IntValue(i));
		record.setField(1, new StringValue("AB" + i + "CD" + i));
		record.setField(3, new DoubleValue(i * 3.141d));
		delegate.setInstance(record);

		int channel = selector.selectChannel(delegate);
		hits[channel]++;
	}

	int totalHitCount = 0;
	for (int hit : hits) {
		assertTrue(hit > 0);
		totalHitCount += hit;
	}
	assertTrue(totalHitCount == numRecords);
}
 

@Test
public void testMultiKeys() {
	final int numberOfChannels = 100;
	final int numRecords = 5000;
	final TypeComparator<Record> multiComp = new RecordComparatorFactory(
		new int[] {0,1, 3}, new Class[] {IntValue.class, StringValue.class, DoubleValue.class}).createComparator();

	final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(
		ShipStrategyType.PARTITION_HASH, multiComp, numberOfChannels);
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());

	int[] hits = new int[numberOfChannels];
	for (int i = 0; i < numRecords; i++) {
		Record record = new Record(4);
		record.setField(0, new IntValue(i));
		record.setField(1, new StringValue("AB" + i + "CD" + i));
		record.setField(3, new DoubleValue(i * 3.141d));
		delegate.setInstance(record);

		int channel = selector.selectChannel(delegate);
		hits[channel]++;
	}

	int totalHitCount = 0;
	for (int hit : hits) {
		assertTrue(hit > 0);
		totalHitCount += hit;
	}
	assertTrue(totalHitCount == numRecords);
}
 

private void assertSelectedChannel(
	ChannelSelector<StringValue> selector,
	StringValue record,
	int expectedChannel) {

	int actualResult = selector.selectChannel(record);
	assertEquals(expectedChannel, actualResult);
}
 

private void assertPartitionHashSelectedChannels(
		ChannelSelector selector,
		SerializationDelegate<Integer> serializationDelegate,
		int record,
		int numberOfChannels) {
	serializationDelegate.setInstance(record);
	int selectedChannel = selector.selectChannel(serializationDelegate);

	assertTrue(selectedChannel >= 0 && selectedChannel <= numberOfChannels - 1);
}
 

private ChannelSelector createChannelSelector(
		ShipStrategyType shipStrategyType,
		TypeComparator comparator,
		int numberOfChannels) {
	final ChannelSelector selector = new OutputEmitter<>(shipStrategyType, comparator);
	selector.setup(numberOfChannels);
	assertEquals(shipStrategyType == ShipStrategyType.BROADCAST, selector.isBroadcast());
	return selector;
}
 

/**
 * Creates the {@link Collector} for the given task, as described by the given configuration. The
 * output collector contains the writers that forward the data to the different tasks that the given task
 * is connected to. Each writer applies the partitioning as described in the configuration.
 *
 * @param task The task that the output collector is created for.
 * @param config The configuration describing the output shipping strategies.
 * @param cl The classloader used to load user defined types.
 * @param eventualOutputs The output writers that this task forwards to the next task for each output.
 * @param outputOffset The offset to start to get the writers for the outputs
 * @param numOutputs The number of outputs described in the configuration.
 *
 * @return The OutputCollector that data produced in this task is submitted to.
 */
public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl,
		List<RecordWriter<?>> eventualOutputs, int outputOffset, int numOutputs) throws Exception
{
	if (numOutputs == 0) {
		return null;
	}

	// get the factory for the serializer
	final TypeSerializerFactory<T> serializerFactory = config.getOutputSerializer(cl);
	final List<RecordWriter<SerializationDelegate<T>>> writers = new ArrayList<>(numOutputs);

	// create a writer for each output
	for (int i = 0; i < numOutputs; i++)
	{
		// create the OutputEmitter from output ship strategy
		final ShipStrategyType strategy = config.getOutputShipStrategy(i);
		final int indexInSubtaskGroup = task.getIndexInSubtaskGroup();
		final TypeComparatorFactory<T> compFactory = config.getOutputComparator(i, cl);

		final ChannelSelector<SerializationDelegate<T>> oe;
		if (compFactory == null) {
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup);
		}
		else {
			final DataDistribution dataDist = config.getOutputDataDistribution(i, cl);
			final Partitioner<?> partitioner = config.getOutputPartitioner(i, cl);

			final TypeComparator<T> comparator = compFactory.createComparator();
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup, comparator, partitioner, dataDist);
		}

		final RecordWriter<SerializationDelegate<T>> recordWriter = new RecordWriterBuilder()
			.setChannelSelector(oe)
			.setTaskName(task.getEnvironment().getTaskInfo().getTaskName())
			.build(task.getEnvironment().getWriter(outputOffset + i));

		recordWriter.setMetricGroup(task.getEnvironment().getMetricGroup().getIOMetricGroup());

		writers.add(recordWriter);
	}
	if (eventualOutputs != null) {
		eventualOutputs.addAll(writers);
	}
	return new OutputCollector<T>(writers, serializerFactory.getSerializer());
}
 

@Test
public void testForcedRebalance() {
	final int numberOfChannels = 100;
	int toTaskIndex = numberOfChannels * 6 / 7;
	int fromTaskIndex = toTaskIndex + numberOfChannels;
	int extraRecords = numberOfChannels / 3;
	int numRecords = 50000 + extraRecords;
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(
		new RecordSerializerFactory().getSerializer());
	final ChannelSelector<SerializationDelegate<Record>> selector = new OutputEmitter<>(
		ShipStrategyType.PARTITION_FORCED_REBALANCE, fromTaskIndex);
	selector.setup(numberOfChannels);

	// Test for IntValue
	int[] hits = getSelectedChannelsHitCount(selector, delegate, RecordType.INTEGER, numRecords, numberOfChannels);
	int totalHitCount = 0;
	for (int i = 0; i < hits.length; i++) {
		if (toTaskIndex <= i || i < toTaskIndex+extraRecords - numberOfChannels) {
			assertTrue(hits[i] == (numRecords / numberOfChannels) + 1);
		} else {
			assertTrue(hits[i] == numRecords / numberOfChannels);
		}
		totalHitCount += hits[i];
	}
	assertTrue(totalHitCount == numRecords);

	toTaskIndex = numberOfChannels / 5;
	fromTaskIndex = toTaskIndex + 2 * numberOfChannels;
	extraRecords = numberOfChannels * 2 / 9;
	numRecords = 10000 + extraRecords;

	// Test for StringValue
	final ChannelSelector<SerializationDelegate<Record>> selector2 = new OutputEmitter<>(
		ShipStrategyType.PARTITION_FORCED_REBALANCE, fromTaskIndex);
	selector2.setup(numberOfChannels);
	hits = getSelectedChannelsHitCount(selector2, delegate, RecordType.STRING, numRecords, numberOfChannels);
	totalHitCount = 0;
	for (int i = 0; i < hits.length; i++) {
		if (toTaskIndex <= i && i < toTaskIndex + extraRecords) {
			assertTrue(hits[i] == (numRecords / numberOfChannels) + 1);
		} else {
			assertTrue(hits[i] == numRecords / numberOfChannels);
		}
		totalHitCount += hits[i];
	}
	assertTrue(totalHitCount == numRecords);
}
 

@Test
public void testForcedRebalance() {
	final int numberOfChannels = 100;
	int toTaskIndex = numberOfChannels * 6 / 7;
	int fromTaskIndex = toTaskIndex + numberOfChannels;
	int extraRecords = numberOfChannels / 3;
	int numRecords = 50000 + extraRecords;
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(
		new RecordSerializerFactory().getSerializer());
	final ChannelSelector<SerializationDelegate<Record>> selector = new OutputEmitter<>(
		ShipStrategyType.PARTITION_FORCED_REBALANCE, fromTaskIndex);
	selector.setup(numberOfChannels);

	// Test for IntValue
	int[] hits = getSelectedChannelsHitCount(selector, delegate, RecordType.INTEGER, numRecords, numberOfChannels);
	int totalHitCount = 0;
	for (int i = 0; i < hits.length; i++) {
		if (toTaskIndex <= i || i < toTaskIndex+extraRecords - numberOfChannels) {
			assertTrue(hits[i] == (numRecords / numberOfChannels) + 1);
		} else {
			assertTrue(hits[i] == numRecords / numberOfChannels);
		}
		totalHitCount += hits[i];
	}
	assertTrue(totalHitCount == numRecords);

	toTaskIndex = numberOfChannels / 5;
	fromTaskIndex = toTaskIndex + 2 * numberOfChannels;
	extraRecords = numberOfChannels * 2 / 9;
	numRecords = 10000 + extraRecords;

	// Test for StringValue
	final ChannelSelector<SerializationDelegate<Record>> selector2 = new OutputEmitter<>(
		ShipStrategyType.PARTITION_FORCED_REBALANCE, fromTaskIndex);
	selector2.setup(numberOfChannels);
	hits = getSelectedChannelsHitCount(selector2, delegate, RecordType.STRING, numRecords, numberOfChannels);
	totalHitCount = 0;
	for (int i = 0; i < hits.length; i++) {
		if (toTaskIndex <= i && i < toTaskIndex + extraRecords) {
			assertTrue(hits[i] == (numRecords / numberOfChannels) + 1);
		} else {
			assertTrue(hits[i] == numRecords / numberOfChannels);
		}
		totalHitCount += hits[i];
	}
	assertTrue(totalHitCount == numRecords);
}
 

@Test
public void testForcedRebalance() {
	final int numberOfChannels = 100;
	int toTaskIndex = numberOfChannels * 6 / 7;
	int fromTaskIndex = toTaskIndex + numberOfChannels;
	int extraRecords = numberOfChannels / 3;
	int numRecords = 50000 + extraRecords;
	final SerializationDelegate<Record> delegate = new SerializationDelegate<>(
		new RecordSerializerFactory().getSerializer());
	final ChannelSelector<SerializationDelegate<Record>> selector = new OutputEmitter<>(
		ShipStrategyType.PARTITION_FORCED_REBALANCE, fromTaskIndex);
	selector.setup(numberOfChannels);

	// Test for IntValue
	int[] hits = getSelectedChannelsHitCount(selector, delegate, RecordType.INTEGER, numRecords, numberOfChannels);
	int totalHitCount = 0;
	for (int i = 0; i < hits.length; i++) {
		if (toTaskIndex <= i || i < toTaskIndex+extraRecords - numberOfChannels) {
			assertTrue(hits[i] == (numRecords / numberOfChannels) + 1);
		} else {
			assertTrue(hits[i] == numRecords / numberOfChannels);
		}
		totalHitCount += hits[i];
	}
	assertTrue(totalHitCount == numRecords);

	toTaskIndex = numberOfChannels / 5;
	fromTaskIndex = toTaskIndex + 2 * numberOfChannels;
	extraRecords = numberOfChannels * 2 / 9;
	numRecords = 10000 + extraRecords;

	// Test for StringValue
	final ChannelSelector<SerializationDelegate<Record>> selector2 = new OutputEmitter<>(
		ShipStrategyType.PARTITION_FORCED_REBALANCE, fromTaskIndex);
	selector2.setup(numberOfChannels);
	hits = getSelectedChannelsHitCount(selector2, delegate, RecordType.STRING, numRecords, numberOfChannels);
	totalHitCount = 0;
	for (int i = 0; i < hits.length; i++) {
		if (toTaskIndex <= i && i < toTaskIndex + extraRecords) {
			assertTrue(hits[i] == (numRecords / numberOfChannels) + 1);
		} else {
			assertTrue(hits[i] == numRecords / numberOfChannels);
		}
		totalHitCount += hits[i];
	}
	assertTrue(totalHitCount == numRecords);
}
 

/**
 * Creates the {@link Collector} for the given task, as described by the given configuration. The
 * output collector contains the writers that forward the data to the different tasks that the given task
 * is connected to. Each writer applies the partitioning as described in the configuration.
 *
 * @param task The task that the output collector is created for.
 * @param config The configuration describing the output shipping strategies.
 * @param cl The classloader used to load user defined types.
 * @param eventualOutputs The output writers that this task forwards to the next task for each output.
 * @param outputOffset The offset to start to get the writers for the outputs
 * @param numOutputs The number of outputs described in the configuration.
 *
 * @return The OutputCollector that data produced in this task is submitted to.
 */
public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl,
		List<RecordWriter<?>> eventualOutputs, int outputOffset, int numOutputs) throws Exception
{
	if (numOutputs == 0) {
		return null;
	}

	// get the factory for the serializer
	final TypeSerializerFactory<T> serializerFactory = config.getOutputSerializer(cl);
	final List<RecordWriter<SerializationDelegate<T>>> writers = new ArrayList<>(numOutputs);

	// create a writer for each output
	for (int i = 0; i < numOutputs; i++)
	{
		// create the OutputEmitter from output ship strategy
		final ShipStrategyType strategy = config.getOutputShipStrategy(i);
		final int indexInSubtaskGroup = task.getIndexInSubtaskGroup();
		final TypeComparatorFactory<T> compFactory = config.getOutputComparator(i, cl);

		final ChannelSelector<SerializationDelegate<T>> oe;
		if (compFactory == null) {
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup);
		}
		else {
			final DataDistribution dataDist = config.getOutputDataDistribution(i, cl);
			final Partitioner<?> partitioner = config.getOutputPartitioner(i, cl);

			final TypeComparator<T> comparator = compFactory.createComparator();
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup, comparator, partitioner, dataDist);
		}

		final RecordWriter<SerializationDelegate<T>> recordWriter = RecordWriter.createRecordWriter(
			task.getEnvironment().getWriter(outputOffset + i),
			oe,
			task.getEnvironment().getTaskInfo().getTaskName());

		recordWriter.setMetricGroup(task.getEnvironment().getMetricGroup().getIOMetricGroup());

		writers.add(recordWriter);
	}
	if (eventualOutputs != null) {
		eventualOutputs.addAll(writers);
	}
	return new OutputCollector<T>(writers, serializerFactory.getSerializer());
}
 

/**
 * Creates the {@link Collector} for the given task, as described by the given configuration. The
 * output collector contains the writers that forward the data to the different tasks that the given task
 * is connected to. Each writer applies the partitioning as described in the configuration.
 *
 * @param task The task that the output collector is created for.
 * @param config The configuration describing the output shipping strategies.
 * @param cl The classloader used to load user defined types.
 * @param eventualOutputs The output writers that this task forwards to the next task for each output.
 * @param outputOffset The offset to start to get the writers for the outputs
 * @param numOutputs The number of outputs described in the configuration.
 *
 * @return The OutputCollector that data produced in this task is submitted to.
 */
public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl,
		List<RecordWriter<?>> eventualOutputs, int outputOffset, int numOutputs) throws Exception
{
	if (numOutputs == 0) {
		return null;
	}

	// get the factory for the serializer
	final TypeSerializerFactory<T> serializerFactory = config.getOutputSerializer(cl);
	final List<RecordWriter<SerializationDelegate<T>>> writers = new ArrayList<>(numOutputs);

	// create a writer for each output
	for (int i = 0; i < numOutputs; i++)
	{
		// create the OutputEmitter from output ship strategy
		final ShipStrategyType strategy = config.getOutputShipStrategy(i);
		final int indexInSubtaskGroup = task.getIndexInSubtaskGroup();
		final TypeComparatorFactory<T> compFactory = config.getOutputComparator(i, cl);

		final ChannelSelector<SerializationDelegate<T>> oe;
		if (compFactory == null) {
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup);
		}
		else {
			final DataDistribution dataDist = config.getOutputDataDistribution(i, cl);
			final Partitioner<?> partitioner = config.getOutputPartitioner(i, cl);

			final TypeComparator<T> comparator = compFactory.createComparator();
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup, comparator, partitioner, dataDist);
		}

		final RecordWriter<SerializationDelegate<T>> recordWriter = new RecordWriterBuilder()
			.setChannelSelector(oe)
			.setTaskName(task.getEnvironment().getTaskInfo().getTaskName())
			.build(task.getEnvironment().getWriter(outputOffset + i));

		recordWriter.setMetricGroup(task.getEnvironment().getMetricGroup().getIOMetricGroup());

		writers.add(recordWriter);
	}
	if (eventualOutputs != null) {
		eventualOutputs.addAll(writers);
	}
	return new OutputCollector<T>(writers, serializerFactory.getSerializer());
}