Java Code Examples for org.apache.flink.runtime.operators.util.TaskConfig#getOutputSerializer()

/**
 * Creates the {@link Collector} for the given task, as described by the given configuration. The
 * output collector contains the writers that forward the data to the different tasks that the given task
 * is connected to. Each writer applies the partitioning as described in the configuration.
 *
 * @param task The task that the output collector is created for.
 * @param config The configuration describing the output shipping strategies.
 * @param cl The classloader used to load user defined types.
 * @param eventualOutputs The output writers that this task forwards to the next task for each output.
 * @param outputOffset The offset to start to get the writers for the outputs
 * @param numOutputs The number of outputs described in the configuration.
 *
 * @return The OutputCollector that data produced in this task is submitted to.
 */
public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl,
		List<RecordWriter<?>> eventualOutputs, int outputOffset, int numOutputs) throws Exception
{
	if (numOutputs == 0) {
		return null;
	}

	// get the factory for the serializer
	final TypeSerializerFactory<T> serializerFactory = config.getOutputSerializer(cl);
	final List<RecordWriter<SerializationDelegate<T>>> writers = new ArrayList<>(numOutputs);

	// create a writer for each output
	for (int i = 0; i < numOutputs; i++)
	{
		// create the OutputEmitter from output ship strategy
		final ShipStrategyType strategy = config.getOutputShipStrategy(i);
		final int indexInSubtaskGroup = task.getIndexInSubtaskGroup();
		final TypeComparatorFactory<T> compFactory = config.getOutputComparator(i, cl);

		final ChannelSelector<SerializationDelegate<T>> oe;
		if (compFactory == null) {
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup);
		}
		else {
			final DataDistribution dataDist = config.getOutputDataDistribution(i, cl);
			final Partitioner<?> partitioner = config.getOutputPartitioner(i, cl);

			final TypeComparator<T> comparator = compFactory.createComparator();
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup, comparator, partitioner, dataDist);
		}

		final RecordWriter<SerializationDelegate<T>> recordWriter = RecordWriter.createRecordWriter(
			task.getEnvironment().getWriter(outputOffset + i),
			oe,
			task.getEnvironment().getTaskInfo().getTaskName());

		recordWriter.setMetricGroup(task.getEnvironment().getMetricGroup().getIOMetricGroup());

		writers.add(recordWriter);
	}
	if (eventualOutputs != null) {
		eventualOutputs.addAll(writers);
	}
	return new OutputCollector<T>(writers, serializerFactory.getSerializer());
}
 

/**
 * Creates the {@link Collector} for the given task, as described by the given configuration. The
 * output collector contains the writers that forward the data to the different tasks that the given task
 * is connected to. Each writer applies the partitioning as described in the configuration.
 *
 * @param task The task that the output collector is created for.
 * @param config The configuration describing the output shipping strategies.
 * @param cl The classloader used to load user defined types.
 * @param eventualOutputs The output writers that this task forwards to the next task for each output.
 * @param outputOffset The offset to start to get the writers for the outputs
 * @param numOutputs The number of outputs described in the configuration.
 *
 * @return The OutputCollector that data produced in this task is submitted to.
 */
public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl,
		List<RecordWriter<?>> eventualOutputs, int outputOffset, int numOutputs) throws Exception
{
	if (numOutputs == 0) {
		return null;
	}

	// get the factory for the serializer
	final TypeSerializerFactory<T> serializerFactory = config.getOutputSerializer(cl);
	final List<RecordWriter<SerializationDelegate<T>>> writers = new ArrayList<>(numOutputs);

	// create a writer for each output
	for (int i = 0; i < numOutputs; i++)
	{
		// create the OutputEmitter from output ship strategy
		final ShipStrategyType strategy = config.getOutputShipStrategy(i);
		final int indexInSubtaskGroup = task.getIndexInSubtaskGroup();
		final TypeComparatorFactory<T> compFactory = config.getOutputComparator(i, cl);

		final ChannelSelector<SerializationDelegate<T>> oe;
		if (compFactory == null) {
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup);
		}
		else {
			final DataDistribution dataDist = config.getOutputDataDistribution(i, cl);
			final Partitioner<?> partitioner = config.getOutputPartitioner(i, cl);

			final TypeComparator<T> comparator = compFactory.createComparator();
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup, comparator, partitioner, dataDist);
		}

		final RecordWriter<SerializationDelegate<T>> recordWriter = new RecordWriterBuilder()
			.setChannelSelector(oe)
			.setTaskName(task.getEnvironment().getTaskInfo().getTaskName())
			.build(task.getEnvironment().getWriter(outputOffset + i));

		recordWriter.setMetricGroup(task.getEnvironment().getMetricGroup().getIOMetricGroup());

		writers.add(recordWriter);
	}
	if (eventualOutputs != null) {
		eventualOutputs.addAll(writers);
	}
	return new OutputCollector<T>(writers, serializerFactory.getSerializer());
}
 

/**
 * Creates the {@link Collector} for the given task, as described by the given configuration. The
 * output collector contains the writers that forward the data to the different tasks that the given task
 * is connected to. Each writer applies the partitioning as described in the configuration.
 *
 * @param task The task that the output collector is created for.
 * @param config The configuration describing the output shipping strategies.
 * @param cl The classloader used to load user defined types.
 * @param eventualOutputs The output writers that this task forwards to the next task for each output.
 * @param outputOffset The offset to start to get the writers for the outputs
 * @param numOutputs The number of outputs described in the configuration.
 *
 * @return The OutputCollector that data produced in this task is submitted to.
 */
public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl,
		List<RecordWriter<?>> eventualOutputs, int outputOffset, int numOutputs) throws Exception
{
	if (numOutputs == 0) {
		return null;
	}

	// get the factory for the serializer
	final TypeSerializerFactory<T> serializerFactory = config.getOutputSerializer(cl);
	final List<RecordWriter<SerializationDelegate<T>>> writers = new ArrayList<>(numOutputs);

	// create a writer for each output
	for (int i = 0; i < numOutputs; i++)
	{
		// create the OutputEmitter from output ship strategy
		final ShipStrategyType strategy = config.getOutputShipStrategy(i);
		final int indexInSubtaskGroup = task.getIndexInSubtaskGroup();
		final TypeComparatorFactory<T> compFactory = config.getOutputComparator(i, cl);

		final ChannelSelector<SerializationDelegate<T>> oe;
		if (compFactory == null) {
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup);
		}
		else {
			final DataDistribution dataDist = config.getOutputDataDistribution(i, cl);
			final Partitioner<?> partitioner = config.getOutputPartitioner(i, cl);

			final TypeComparator<T> comparator = compFactory.createComparator();
			oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup, comparator, partitioner, dataDist);
		}

		final RecordWriter<SerializationDelegate<T>> recordWriter = new RecordWriterBuilder()
			.setChannelSelector(oe)
			.setTaskName(task.getEnvironment().getTaskInfo().getTaskName())
			.build(task.getEnvironment().getWriter(outputOffset + i));

		recordWriter.setMetricGroup(task.getEnvironment().getMetricGroup().getIOMetricGroup());

		writers.add(recordWriter);
	}
	if (eventualOutputs != null) {
		eventualOutputs.addAll(writers);
	}
	return new OutputCollector<T>(writers, serializerFactory.getSerializer());
}