org.apache.flink.streaming.api.graph.StreamEdge Java Examples

public void finish() {

		List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
		outEdgesInOrder.add(
			new StreamEdge(
				new StreamNode(null, chainIndex, null, null, null, null, null, null),
				new StreamNode(null, chainIndex , null, null, null, null, null, null),
				0,
				Collections.<String>emptyList(),
				new BroadcastPartitioner<Object>(),
				null));

		tailConfig.setBufferTimeout(0);
		tailConfig.setChainEnd();
		tailConfig.setOutputSelectors(Collections.emptyList());
		tailConfig.setNumberOfOutputs(1);
		tailConfig.setOutEdgesInOrder(outEdgesInOrder);
		tailConfig.setNonChainedOutputs(outEdgesInOrder);
		headConfig.setTransitiveChainedTaskConfigs(chainedConfigs);
		headConfig.setOutEdgesInOrder(outEdgesInOrder);
	}
 

/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public void setupOutputForSingletonOperatorChain() {
	Preconditions.checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setBufferTimeout(0);
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);
	streamConfig.setOperatorID(new OperatorID(4711L, 123L));

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);
}
 

public void finish() {

		List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
		outEdgesInOrder.add(
			new StreamEdge(
				new StreamNode(chainIndex, null, null, (StreamOperator<?>) null, null, null, null),
				new StreamNode(chainIndex , null, null, (StreamOperator<?>) null, null, null, null),
				0,
				Collections.<String>emptyList(),
				new BroadcastPartitioner<Object>(),
				null));

		tailConfig.setBufferTimeout(0);
		tailConfig.setChainEnd();
		tailConfig.setOutputSelectors(Collections.emptyList());
		tailConfig.setNumberOfOutputs(1);
		tailConfig.setOutEdgesInOrder(outEdgesInOrder);
		tailConfig.setNonChainedOutputs(outEdgesInOrder);
		headConfig.setTransitiveChainedTaskConfigs(chainedConfigs);
		headConfig.setOutEdgesInOrder(outEdgesInOrder);
	}
 

private static <OUT> List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createRecordWriters(
		StreamConfig configuration,
		Environment environment) {
	List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> recordWriters = new ArrayList<>();
	List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader());
	Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader());

	for (int i = 0; i < outEdgesInOrder.size(); i++) {
		StreamEdge edge = outEdgesInOrder.get(i);
		recordWriters.add(
			createRecordWriter(
				edge,
				i,
				environment,
				environment.getTaskInfo().getTaskName(),
				chainedConfigs.get(edge.getSourceId()).getBufferTimeout()));
	}
	return recordWriters;
}
 

private RecordWriterOutput<OUT> createStreamOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		StreamEdge edge,
		StreamConfig upStreamConfig,
		Environment taskEnvironment) {
	OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput

	TypeSerializer outSerializer = null;

	if (edge.getOutputTag() != null) {
		// side output
		outSerializer = upStreamConfig.getTypeSerializerSideOut(
				edge.getOutputTag(), taskEnvironment.getUserClassLoader());
	} else {
		// main output
		outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader());
	}

	return new RecordWriterOutput<>(recordWriter, outSerializer, sideOutputTag, this);
}
 

private RecordWriterOutput<OUT> createStreamOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		StreamEdge edge,
		StreamConfig upStreamConfig,
		Environment taskEnvironment) {
	OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput

	TypeSerializer outSerializer = null;

	if (edge.getOutputTag() != null) {
		// side output
		outSerializer = upStreamConfig.getTypeSerializerSideOut(
				edge.getOutputTag(), taskEnvironment.getUserClassLoader());
	} else {
		// main output
		outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader());
	}

	return new RecordWriterOutput<>(recordWriter, outSerializer, sideOutputTag, this);
}
 

@VisibleForTesting
public static <OUT> List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createRecordWriters(
		StreamConfig configuration,
		Environment environment) {
	List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> recordWriters = new ArrayList<>();
	List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader());
	Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader());

	for (int i = 0; i < outEdgesInOrder.size(); i++) {
		StreamEdge edge = outEdgesInOrder.get(i);
		recordWriters.add(
			createRecordWriter(
				edge,
				i,
				environment,
				environment.getTaskInfo().getTaskName(),
				chainedConfigs.get(edge.getSourceId()).getBufferTimeout()));
	}
	return recordWriters;
}
 

/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public void setupOutputForSingletonOperatorChain() {
	Preconditions.checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setBufferTimeout(0);
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);
	streamConfig.setOperatorID(new OperatorID(4711L, 123L));

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	StreamNode sourceVertexDummy = new StreamNode(null, 0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(null, 1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);
}
 

/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public void setupOutputForSingletonOperatorChain() {
	Preconditions.checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);
	streamConfig.setOperatorID(new OperatorID(4711L, 123L));

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<>();
	StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", new LinkedList<>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", new LinkedList<>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<>(), new BroadcastPartitioner<>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);
}
 

private RecordWriterOutput<OUT> createStreamOutput(
		RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter,
		StreamEdge edge,
		StreamConfig upStreamConfig,
		Environment taskEnvironment) {
	OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput

	TypeSerializer outSerializer = null;

	if (edge.getOutputTag() != null) {
		// side output
		outSerializer = upStreamConfig.getTypeSerializerSideOut(
				edge.getOutputTag(), taskEnvironment.getUserClassLoader());
	} else {
		// main output
		outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader());
	}

	return new RecordWriterOutput<>(recordWriter, outSerializer, sideOutputTag, this);
}
 

public OWNER finish() {
	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	outEdgesInOrder.add(
		new StreamEdge(
			new StreamNode(chainIndex, null, null, (StreamOperator<?>) null, null, null, null),
			new StreamNode(chainIndex , null, null, (StreamOperator<?>) null, null, null, null),
			0,
			Collections.<String>emptyList(),
			new BroadcastPartitioner<Object>(),
			null));

	tailConfig.setChainEnd();
	tailConfig.setOutputSelectors(Collections.emptyList());
	tailConfig.setNumberOfOutputs(1);
	tailConfig.setOutEdgesInOrder(outEdgesInOrder);
	tailConfig.setNonChainedOutputs(outEdgesInOrder);
	headConfig.setTransitiveChainedTaskConfigs(chainedConfigs);
	headConfig.setOutEdgesInOrder(outEdgesInOrder);

	return owner;
}
 

private static <OUT> RecordWriter<SerializationDelegate<StreamRecord<OUT>>> createRecordWriter(
		StreamEdge edge,
		int outputIndex,
		Environment environment,
		String taskName,
		long bufferTimeout) {
	@SuppressWarnings("unchecked")
	StreamPartitioner<OUT> outputPartitioner = (StreamPartitioner<OUT>) edge.getPartitioner();

	LOG.debug("Using partitioner {} for output {} of task {}", outputPartitioner, outputIndex, taskName);

	ResultPartitionWriter bufferWriter = environment.getWriter(outputIndex);

	// we initialize the partitioner here with the number of key groups (aka max. parallelism)
	if (outputPartitioner instanceof ConfigurableStreamPartitioner) {
		int numKeyGroups = bufferWriter.getNumTargetKeyGroups();
		if (0 < numKeyGroups) {
			((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups);
		}
	}

	RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output =
		RecordWriter.createRecordWriter(bufferWriter, outputPartitioner, bufferTimeout, taskName);
	output.setMetricGroup(environment.getMetricGroup().getIOMetricGroup());
	return output;
}
 

@VisibleForTesting
public static <OUT> List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createRecordWriters(
		StreamConfig configuration,
		Environment environment) {
	List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> recordWriters = new ArrayList<>();
	List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader());
	Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader());

	for (int i = 0; i < outEdgesInOrder.size(); i++) {
		StreamEdge edge = outEdgesInOrder.get(i);
		recordWriters.add(
			createRecordWriter(
				edge,
				i,
				environment,
				environment.getTaskInfo().getTaskName(),
				chainedConfigs.get(edge.getSourceId()).getBufferTimeout()));
	}
	return recordWriters;
}
 

public MockStreamConfig(Configuration configuration, int numberOfOutputs) {
	super(configuration);

	setChainStart();
	setOutputSelectors(Collections.emptyList());
	setNumberOfOutputs(numberOfOutputs);
	setTypeSerializerOut(new StringSerializer());
	setVertexID(0);
	setStreamOperator(new TestSequentialReadingStreamOperator("test operator"));
	setOperatorID(new OperatorID());

	StreamOperator dummyOperator = new AbstractStreamOperator() {
		private static final long serialVersionUID = 1L;
	};

	StreamNode sourceVertex = new StreamNode(0, null, null, dummyOperator, "source", new ArrayList<>(), SourceStreamTask.class);
	StreamNode targetVertex = new StreamNode(1, null, null, dummyOperator, "target", new ArrayList<>(), SourceStreamTask.class);

	List<StreamEdge> outEdgesInOrder = new ArrayList<>(numberOfOutputs);
	for (int i = 0; i < numberOfOutputs; i++) {
		outEdgesInOrder.add(
			new StreamEdge(sourceVertex, targetVertex, numberOfOutputs, new ArrayList<>(), new BroadcastPartitioner<>(), null));
	}
	setOutEdgesInOrder(outEdgesInOrder);
	setNonChainedOutputs(outEdgesInOrder);
}
 

@Override
protected void initializeInputs(StreamMockEnvironment streamMockEnvironment) {
	inputGates = new StreamTestSingleInputGate[inputSerializers.size()];
	List<StreamEdge> inPhysicalEdges = new LinkedList<>();

	StreamOperator<?> dummyOperator = new AbstractStreamOperator<Object>() {
		private static final long serialVersionUID = 1L;
	};

	StreamNode sourceVertexDummy = new StreamNode(0, "default group", null, dummyOperator, "source dummy", new LinkedList<>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "default group", null, dummyOperator, "target dummy", new LinkedList<>(), SourceStreamTask.class);

	for (int i = 0; i < inputSerializers.size(); i++) {
		TypeSerializer<?> inputSerializer = inputSerializers.get(i);
		inputGates[i] = new StreamTestSingleInputGate<>(
			inputChannelsPerGate.get(i),
			i,
			inputSerializer,
			bufferSize);

		StreamEdge streamEdge = new StreamEdge(
			sourceVertexDummy,
			targetVertexDummy,
			i + 1,
			new LinkedList<>(),
			new BroadcastPartitioner<>(),
			null);

		inPhysicalEdges.add(streamEdge);
		streamMockEnvironment.addInputGate(inputGates[i].getInputGate());
	}

	streamConfig.setInPhysicalEdges(inPhysicalEdges);
	streamConfig.setNumberOfInputs(inputGates.length);
	streamConfig.setTypeSerializersIn(inputSerializers.toArray(new TypeSerializer[inputSerializers.size()]));
}
 

private static boolean isCustomPartitioned(List<StreamEdge> edges) {
	boolean result = true;
	for (StreamEdge edge: edges) {
		if (!(edge.getPartitioner() instanceof CustomPartitionerWrapper)) {
			result = false;
		}
	}
	return result;
}
 

private static boolean isCustomPartitioned(List<StreamEdge> edges) {
	boolean result = true;
	for (StreamEdge edge: edges) {
		if (!(edge.getPartitioner() instanceof CustomPartitionerWrapper)) {
			result = false;
		}
	}
	return result;
}
 

private static boolean isPartitioned(List<StreamEdge> edges) {
	boolean result = true;
	for (StreamEdge edge: edges) {
		if (!(edge.getPartitioner() instanceof KeyGroupStreamPartitioner)) {
			result = false;
		}
	}
	return result;
}
 

@Override
public void init() throws Exception {
	StreamConfig configuration = getConfiguration();
	ClassLoader userClassLoader = getUserCodeClassLoader();

	TypeSerializer<?>[] inputDeserializers = configuration.getTypeSerializersIn(userClassLoader);

	ArrayList<IndexedInputGate>[] inputLists = new ArrayList[inputDeserializers.length];
	WatermarkGauge[] watermarkGauges = new WatermarkGauge[inputDeserializers.length];

	for (int i = 0; i < inputDeserializers.length; i++) {
		inputLists[i] = new ArrayList<>();
		watermarkGauges[i] = new WatermarkGauge();
		headOperator.getMetricGroup().gauge(MetricNames.currentInputWatermarkName(i + 1), watermarkGauges[i]);
	}

	MinWatermarkGauge minInputWatermarkGauge = new MinWatermarkGauge(watermarkGauges);
	headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge);

	List<StreamEdge> inEdges = configuration.getInPhysicalEdges(userClassLoader);
	int numberOfInputs = configuration.getNumberOfInputs();

	for (int i = 0; i < numberOfInputs; i++) {
		int inputType = inEdges.get(i).getTypeNumber();
		IndexedInputGate reader = getEnvironment().getInputGate(i);
		inputLists[inputType - 1].add(reader);
	}

	createInputProcessor(inputLists, inputDeserializers, watermarkGauges);

	// wrap watermark gauge since registered metrics must be unique
	getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge::getValue);
}
 

/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public StreamTaskMailboxTestHarnessBuilder<OUT> setupOutputForSingletonOperatorChain(
		StreamOperatorFactory<?> factory,
		OperatorID operatorID) {
	checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);

	streamConfig.setStreamOperatorFactory(factory);
	streamConfig.setOperatorID(operatorID);

	return this;
}
 

@Override
public void init() throws Exception {
	StreamConfig configuration = getConfiguration();
	ClassLoader userClassLoader = getUserCodeClassLoader();

	TypeSerializer<IN1> inputDeserializer1 = configuration.getTypeSerializerIn1(userClassLoader);
	TypeSerializer<IN2> inputDeserializer2 = configuration.getTypeSerializerIn2(userClassLoader);

	int numberOfInputs = configuration.getNumberOfInputs();

	ArrayList<IndexedInputGate> inputList1 = new ArrayList<>();
	ArrayList<IndexedInputGate> inputList2 = new ArrayList<>();

	List<StreamEdge> inEdges = configuration.getInPhysicalEdges(userClassLoader);

	for (int i = 0; i < numberOfInputs; i++) {
		int inputType = inEdges.get(i).getTypeNumber();
		IndexedInputGate reader = getEnvironment().getInputGate(i);
		switch (inputType) {
			case 1:
				inputList1.add(reader);
				break;
			case 2:
				inputList2.add(reader);
				break;
			default:
				throw new RuntimeException("Invalid input type number: " + inputType);
		}
	}

	createInputProcessor(inputList1, inputList2, inputDeserializer1, inputDeserializer2);

	headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge);
	headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_1_WATERMARK, input1WatermarkGauge);
	headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_2_WATERMARK, input2WatermarkGauge);
	// wrap watermark gauge since registered metrics must be unique
	getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge::getValue);
}
 

private static <OUT> RecordWriter<SerializationDelegate<StreamRecord<OUT>>> createRecordWriter(
		StreamEdge edge,
		int outputIndex,
		Environment environment,
		String taskName,
		long bufferTimeout) {
	@SuppressWarnings("unchecked")
	StreamPartitioner<OUT> outputPartitioner = (StreamPartitioner<OUT>) edge.getPartitioner();

	LOG.debug("Using partitioner {} for output {} of task {}", outputPartitioner, outputIndex, taskName);

	ResultPartitionWriter bufferWriter = environment.getWriter(outputIndex);

	// we initialize the partitioner here with the number of key groups (aka max. parallelism)
	if (outputPartitioner instanceof ConfigurableStreamPartitioner) {
		int numKeyGroups = bufferWriter.getNumTargetKeyGroups();
		if (0 < numKeyGroups) {
			((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups);
		}
	}

	RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output = new RecordWriterBuilder<SerializationDelegate<StreamRecord<OUT>>>()
		.setChannelSelector(outputPartitioner)
		.setTimeout(bufferTimeout)
		.setTaskName(taskName)
		.build(bufferWriter);
	output.setMetricGroup(environment.getMetricGroup().getIOMetricGroup());
	return output;
}
 

private static boolean isPartitioned(List<StreamEdge> edges) {
	boolean result = true;
	for (StreamEdge edge: edges) {
		if (!(edge.getPartitioner() instanceof KeyGroupStreamPartitioner)) {
			result = false;
		}
	}
	return result;
}
 

private <IN, OUT> WatermarkGaugeExposingOutput<StreamRecord<IN>> createChainedOperator(
		StreamTask<?, ?> containingTask,
		StreamConfig operatorConfig,
		Map<Integer, StreamConfig> chainedConfigs,
		ClassLoader userCodeClassloader,
		Map<StreamEdge, RecordWriterOutput<?>> streamOutputs,
		List<StreamOperator<?>> allOperators,
		OutputTag<IN> outputTag) {
	// create the output that the operator writes to first. this may recursively create more operators
	WatermarkGaugeExposingOutput<StreamRecord<OUT>> chainedOperatorOutput = createOutputCollector(
		containingTask,
		operatorConfig,
		chainedConfigs,
		userCodeClassloader,
		streamOutputs,
		allOperators);

	// now create the operator and give it the output collector to write its output to
	StreamOperatorFactory<OUT> chainedOperatorFactory = operatorConfig.getStreamOperatorFactory(userCodeClassloader);
	OneInputStreamOperator<IN, OUT> chainedOperator = chainedOperatorFactory.createStreamOperator(
			containingTask, operatorConfig, chainedOperatorOutput);

	allOperators.add(chainedOperator);

	WatermarkGaugeExposingOutput<StreamRecord<IN>> currentOperatorOutput;
	if (containingTask.getExecutionConfig().isObjectReuseEnabled()) {
		currentOperatorOutput = new ChainingOutput<>(chainedOperator, this, outputTag);
	}
	else {
		TypeSerializer<IN> inSerializer = operatorConfig.getTypeSerializerIn1(userCodeClassloader);
		currentOperatorOutput = new CopyingChainingOutput<>(chainedOperator, inSerializer, outputTag, this);
	}

	// wrap watermark gauges since registered metrics must be unique
	chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, currentOperatorOutput.getWatermarkGauge()::getValue);
	chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_OUTPUT_WATERMARK, chainedOperatorOutput.getWatermarkGauge()::getValue);

	return currentOperatorOutput;
}
 

@Override
public void init() throws Exception {
	StreamConfig configuration = getConfiguration();
	ClassLoader userClassLoader = getUserCodeClassLoader();

	TypeSerializer<IN1> inputDeserializer1 = configuration.getTypeSerializerIn1(userClassLoader);
	TypeSerializer<IN2> inputDeserializer2 = configuration.getTypeSerializerIn2(userClassLoader);

	int numberOfInputs = configuration.getNumberOfInputs();

	ArrayList<InputGate> inputList1 = new ArrayList<InputGate>();
	ArrayList<InputGate> inputList2 = new ArrayList<InputGate>();

	List<StreamEdge> inEdges = configuration.getInPhysicalEdges(userClassLoader);

	for (int i = 0; i < numberOfInputs; i++) {
		int inputType = inEdges.get(i).getTypeNumber();
		InputGate reader = getEnvironment().getInputGate(i);
		switch (inputType) {
			case 1:
				inputList1.add(reader);
				break;
			case 2:
				inputList2.add(reader);
				break;
			default:
				throw new RuntimeException("Invalid input type number: " + inputType);
		}
	}

	createInputProcessor(inputList1, inputList2, inputDeserializer1, inputDeserializer2);

	headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge);
	headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_1_WATERMARK, input1WatermarkGauge);
	headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_2_WATERMARK, input2WatermarkGauge);
	// wrap watermark gauge since registered metrics must be unique
	getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, minInputWatermarkGauge::getValue);
}
 

private <IN, OUT> WatermarkGaugeExposingOutput<StreamRecord<IN>> createChainedOperator(
		StreamTask<?, ?> containingTask,
		StreamConfig operatorConfig,
		Map<Integer, StreamConfig> chainedConfigs,
		ClassLoader userCodeClassloader,
		Map<StreamEdge, RecordWriterOutput<?>> streamOutputs,
		List<StreamOperator<?>> allOperators,
		OutputTag<IN> outputTag) {
	// create the output that the operator writes to first. this may recursively create more operators
	WatermarkGaugeExposingOutput<StreamRecord<OUT>> chainedOperatorOutput = createOutputCollector(
		containingTask,
		operatorConfig,
		chainedConfigs,
		userCodeClassloader,
		streamOutputs,
		allOperators);

	// now create the operator and give it the output collector to write its output to
	OneInputStreamOperator<IN, OUT> chainedOperator = operatorConfig.getStreamOperator(userCodeClassloader);

	chainedOperator.setup(containingTask, operatorConfig, chainedOperatorOutput);

	allOperators.add(chainedOperator);

	WatermarkGaugeExposingOutput<StreamRecord<IN>> currentOperatorOutput;
	if (containingTask.getExecutionConfig().isObjectReuseEnabled()) {
		currentOperatorOutput = new ChainingOutput<>(chainedOperator, this, outputTag);
	}
	else {
		TypeSerializer<IN> inSerializer = operatorConfig.getTypeSerializerIn1(userCodeClassloader);
		currentOperatorOutput = new CopyingChainingOutput<>(chainedOperator, inSerializer, outputTag, this);
	}

	// wrap watermark gauges since registered metrics must be unique
	chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, currentOperatorOutput.getWatermarkGauge()::getValue);
	chainedOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_OUTPUT_WATERMARK, chainedOperatorOutput.getWatermarkGauge()::getValue);

	return currentOperatorOutput;
}
 

private static <OUT> RecordWriter<SerializationDelegate<StreamRecord<OUT>>> createRecordWriter(
		StreamEdge edge,
		int outputIndex,
		Environment environment,
		String taskName,
		long bufferTimeout) {
	@SuppressWarnings("unchecked")
	StreamPartitioner<OUT> outputPartitioner = (StreamPartitioner<OUT>) edge.getPartitioner();

	LOG.debug("Using partitioner {} for output {} of task {}", outputPartitioner, outputIndex, taskName);

	ResultPartitionWriter bufferWriter = environment.getWriter(outputIndex);

	// we initialize the partitioner here with the number of key groups (aka max. parallelism)
	if (outputPartitioner instanceof ConfigurableStreamPartitioner) {
		int numKeyGroups = bufferWriter.getNumTargetKeyGroups();
		if (0 < numKeyGroups) {
			((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups);
		}
	}

	RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output = new RecordWriterBuilder()
		.setChannelSelector(outputPartitioner)
		.setTimeout(bufferTimeout)
		.setTaskName(taskName)
		.build(bufferWriter);
	output.setMetricGroup(environment.getMetricGroup().getIOMetricGroup());
	return output;
}
 

private static boolean isPartitioned(List<StreamEdge> edges) {
	boolean result = true;
	for (StreamEdge edge: edges) {
		if (!(edge.getPartitioner() instanceof KeyGroupStreamPartitioner)) {
			result = false;
		}
	}
	return result;
}
 

private static boolean isCustomPartitioned(List<StreamEdge> edges) {
	boolean result = true;
	for (StreamEdge edge: edges) {
		if (!(edge.getPartitioner() instanceof CustomPartitionerWrapper)) {
			result = false;
		}
	}
	return result;
}
 

@SuppressWarnings({"unchecked", "rawtypes"})
public CopyingDirectedOutput(
		List<OutputSelector<OUT>> outputSelectors,
		List<? extends Tuple2<? extends Output<StreamRecord<OUT>>, StreamEdge>> outputs) {
	super(outputSelectors, outputs);
}