org.apache.flink.runtime.io.network.partition.consumer.UnionInputGate Java Examples

private InputGate createInputGate(TaskManagerLocation senderLocation) throws Exception {
	InputGate[] gates = new InputGate[channels];
	for (int channel = 0; channel < channels; ++channel) {
		final InputGateDeploymentDescriptor gateDescriptor = createInputGateDeploymentDescriptor(
			senderLocation,
			channel,
			location);

		final InputGate gate = createInputGateWithMetrics(gateFactory, gateDescriptor, channel);

		gate.setup();
		gates[channel] = gate;
	}

	if (channels > 1) {
		return new UnionInputGate(gates);
	} else {
		return gates[0];
	}
}
 

private InputGate createInputGate(TaskManagerLocation senderLocation) throws Exception {
	IndexedInputGate[] gates = new IndexedInputGate[partitionIds.length];
	for (int gateIndex = 0; gateIndex < gates.length; ++gateIndex) {
		final InputGateDeploymentDescriptor gateDescriptor = createInputGateDeploymentDescriptor(
			senderLocation,
			gateIndex,
			location);

		final IndexedInputGate gate = createInputGateWithMetrics(gateFactory, gateDescriptor, gateIndex);

		gate.setup();
		gates[gateIndex] = gate;
	}

	if (gates.length > 1) {
		return new UnionInputGate(gates);
	} else {
		return gates[0];
	}
}
 

/**
 * Creates the record readers for the number of inputs as defined by {@link #getNumTaskInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initInputReaders() throws Exception {
	final int numInputs = getNumTaskInputs();
	final MutableReader<?>[] inputReaders = new MutableReader<?>[numInputs];

	int currentReaderOffset = 0;

	for (int i = 0; i < numInputs; i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getGroupSize(i);

		if (groupSize == 1) {
			// non-union case
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			InputGate[] readers = new InputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.inputReaders = inputReaders;

	// final sanity check
	if (currentReaderOffset != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 

/**
 * Creates the record readers for the extra broadcast inputs as configured by {@link TaskConfig#getNumBroadcastInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initBroadcastInputReaders() throws Exception {
	final int numBroadcastInputs = this.config.getNumBroadcastInputs();
	final MutableReader<?>[] broadcastInputReaders = new MutableReader<?>[numBroadcastInputs];

	int currentReaderOffset = config.getNumInputs();

	for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getBroadcastGroupSize(i);
		if (groupSize == 1) {
			// non-union case
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			InputGate[] readers = new InputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.broadcastInputReaders = broadcastInputReaders;
}
 

/**
 * Initializes the input readers of the DataSinkTask.
 * 
 * @throws RuntimeException
 *         Thrown in case of invalid task input configuration.
 */
@SuppressWarnings("unchecked")
private void initInputReaders() throws Exception {
	int numGates = 0;
	//  ---------------- create the input readers ---------------------
	// in case where a logical input unions multiple physical inputs, create a union reader
	final int groupSize = this.config.getGroupSize(0);
	numGates += groupSize;
	if (groupSize == 1) {
		// non-union case
		inputReader = new MutableRecordReader<DeserializationDelegate<IT>>(
				getEnvironment().getInputGate(0),
				getEnvironment().getTaskManagerInfo().getTmpDirectories());
	} else if (groupSize > 1){
		// union case
		inputReader = new MutableRecordReader<IOReadableWritable>(
				new UnionInputGate(getEnvironment().getAllInputGates()),
				getEnvironment().getTaskManagerInfo().getTmpDirectories());
	} else {
		throw new Exception("Illegal input group size in task configuration: " + groupSize);
	}
	
	this.inputTypeSerializerFactory = this.config.getInputSerializer(0, getUserCodeClassLoader());
	@SuppressWarnings({ "rawtypes" })
	final MutableObjectIterator<?> iter = new ReaderIterator(inputReader, this.inputTypeSerializerFactory.getSerializer());
	this.reader = (MutableObjectIterator<IT>)iter;

	// final sanity check
	if (numGates != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 

public static InputGate createInputGate(InputGate[] inputGates) {
	if (inputGates.length <= 0) {
		throw new RuntimeException("No such input gate.");
	}

	if (inputGates.length < 2) {
		return inputGates[0];
	} else {
		return new UnionInputGate(inputGates);
	}
}
 

/**
 * Creates the record readers for the number of inputs as defined by {@link #getNumTaskInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initInputReaders() throws Exception {
	final int numInputs = getNumTaskInputs();
	final MutableReader<?>[] inputReaders = new MutableReader<?>[numInputs];

	int currentReaderOffset = 0;

	for (int i = 0; i < numInputs; i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getGroupSize(i);

		if (groupSize == 1) {
			// non-union case
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			InputGate[] readers = new InputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.inputReaders = inputReaders;

	// final sanity check
	if (currentReaderOffset != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 

/**
 * Creates the record readers for the extra broadcast inputs as configured by {@link TaskConfig#getNumBroadcastInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initBroadcastInputReaders() throws Exception {
	final int numBroadcastInputs = this.config.getNumBroadcastInputs();
	final MutableReader<?>[] broadcastInputReaders = new MutableReader<?>[numBroadcastInputs];

	int currentReaderOffset = config.getNumInputs();

	for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getBroadcastGroupSize(i);
		if (groupSize == 1) {
			// non-union case
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			InputGate[] readers = new InputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.broadcastInputReaders = broadcastInputReaders;
}
 

/**
 * Initializes the input readers of the DataSinkTask.
 * 
 * @throws RuntimeException
 *         Thrown in case of invalid task input configuration.
 */
@SuppressWarnings("unchecked")
private void initInputReaders() throws Exception {
	int numGates = 0;
	//  ---------------- create the input readers ---------------------
	// in case where a logical input unions multiple physical inputs, create a union reader
	final int groupSize = this.config.getGroupSize(0);
	numGates += groupSize;
	if (groupSize == 1) {
		// non-union case
		inputReader = new MutableRecordReader<DeserializationDelegate<IT>>(
				getEnvironment().getInputGate(0),
				getEnvironment().getTaskManagerInfo().getTmpDirectories());
	} else if (groupSize > 1){
		// union case
		inputReader = new MutableRecordReader<IOReadableWritable>(
				new UnionInputGate(getEnvironment().getAllInputGates()),
				getEnvironment().getTaskManagerInfo().getTmpDirectories());
	} else {
		throw new Exception("Illegal input group size in task configuration: " + groupSize);
	}
	
	this.inputTypeSerializerFactory = this.config.getInputSerializer(0, getUserCodeClassLoader());
	@SuppressWarnings({ "rawtypes" })
	final MutableObjectIterator<?> iter = new ReaderIterator(inputReader, this.inputTypeSerializerFactory.getSerializer());
	this.reader = (MutableObjectIterator<IT>)iter;

	// final sanity check
	if (numGates != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 

public static InputGate createInputGate(InputGate[] inputGates) {
	if (inputGates.length <= 0) {
		throw new RuntimeException("No such input gate.");
	}

	if (inputGates.length < 2) {
		return inputGates[0];
	} else {
		return new UnionInputGate(inputGates);
	}
}
 

/**
 * Creates the record readers for the number of inputs as defined by {@link #getNumTaskInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initInputReaders() throws Exception {
	final int numInputs = getNumTaskInputs();
	final MutableReader<?>[] inputReaders = new MutableReader<?>[numInputs];

	int currentReaderOffset = 0;

	for (int i = 0; i < numInputs; i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getGroupSize(i);

		if (groupSize == 1) {
			// non-union case
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			IndexedInputGate[] readers = new IndexedInputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.inputReaders = inputReaders;

	// final sanity check
	if (currentReaderOffset != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 

/**
 * Creates the record readers for the extra broadcast inputs as configured by {@link TaskConfig#getNumBroadcastInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initBroadcastInputReaders() throws Exception {
	final int numBroadcastInputs = this.config.getNumBroadcastInputs();
	final MutableReader<?>[] broadcastInputReaders = new MutableReader<?>[numBroadcastInputs];

	int currentReaderOffset = config.getNumInputs();

	for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getBroadcastGroupSize(i);
		if (groupSize == 1) {
			// non-union case
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			IndexedInputGate[] readers = new IndexedInputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.broadcastInputReaders = broadcastInputReaders;
}
 

/**
 * Initializes the input readers of the DataSinkTask.
 * 
 * @throws RuntimeException
 *         Thrown in case of invalid task input configuration.
 */
@SuppressWarnings("unchecked")
private void initInputReaders() throws Exception {
	int numGates = 0;
	//  ---------------- create the input readers ---------------------
	// in case where a logical input unions multiple physical inputs, create a union reader
	final int groupSize = this.config.getGroupSize(0);
	numGates += groupSize;
	if (groupSize == 1) {
		// non-union case
		inputReader = new MutableRecordReader<DeserializationDelegate<IT>>(
				getEnvironment().getInputGate(0),
				getEnvironment().getTaskManagerInfo().getTmpDirectories());
	} else if (groupSize > 1){
		// union case
		inputReader = new MutableRecordReader<IOReadableWritable>(
				new UnionInputGate(getEnvironment().getAllInputGates()),
				getEnvironment().getTaskManagerInfo().getTmpDirectories());
	} else {
		throw new Exception("Illegal input group size in task configuration: " + groupSize);
	}
	
	this.inputTypeSerializerFactory = this.config.getInputSerializer(0, getUserCodeClassLoader());
	@SuppressWarnings({ "rawtypes" })
	final MutableObjectIterator<?> iter = new ReaderIterator(inputReader, this.inputTypeSerializerFactory.getSerializer());
	this.reader = (MutableObjectIterator<IT>)iter;

	// final sanity check
	if (numGates != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 

public static InputGate createInputGate(List<IndexedInputGate> inputGates) {
	if (inputGates.size() <= 0) {
		throw new RuntimeException("No such input gate.");
	}

	if (inputGates.size() == 1) {
		return inputGates.get(0);
	} else {
		return new UnionInputGate(inputGates.toArray(new IndexedInputGate[0]));
	}
}
 

private InputGate createInputGate(
		JobID jobId,
		IntermediateDataSetID dataSetID,
		ExecutionAttemptID executionAttemptID,
		final TaskManagerLocation senderLocation,
		NetworkEnvironment environment,
		final int channels) throws IOException {

	InputGate[] gates = new InputGate[channels];
	for (int channel = 0; channel < channels; ++channel) {
		int finalChannel = channel;
		InputChannelDeploymentDescriptor[] channelDescriptors = Arrays.stream(partitionIds)
			.map(partitionId -> new InputChannelDeploymentDescriptor(
				partitionId,
				localMode ? ResultPartitionLocation.createLocal() : ResultPartitionLocation.createRemote(new ConnectionID(senderLocation, finalChannel))))
			.toArray(InputChannelDeploymentDescriptor[]::new);

		final InputGateDeploymentDescriptor gateDescriptor = new InputGateDeploymentDescriptor(
			dataSetID,
			ResultPartitionType.PIPELINED_BOUNDED,
			channel,
			channelDescriptors);

		SingleInputGate gate = SingleInputGate.create(
			"receiving task[" + channel + "]",
			jobId,
			executionAttemptID,
			gateDescriptor,
			environment,
			new NoOpTaskActions(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup());

		environment.setupInputGate(gate);
		gates[channel] = gate;
	}

	if (channels > 1) {
		return new UnionInputGate(gates);
	} else {
		return gates[0];
	}
}