org.apache.flink.runtime.io.network.api.reader.MutableReader Java Examples

protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
 

protected MutableObjectIterator<?> createInputIterator(MutableReader<?> inputReader, TypeSerializerFactory<?> serializerFactory) {
	@SuppressWarnings("unchecked")
	MutableReader<DeserializationDelegate<?>> reader = (MutableReader<DeserializationDelegate<?>>) inputReader;
	@SuppressWarnings({ "unchecked", "rawtypes" })
	final MutableObjectIterator<?> iter = new ReaderIterator(reader, serializerFactory.getSerializer());
	return iter;
}
 

/**
 * Creates the record readers for the extra broadcast inputs as configured by {@link TaskConfig#getNumBroadcastInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initBroadcastInputReaders() throws Exception {
	final int numBroadcastInputs = this.config.getNumBroadcastInputs();
	final MutableReader<?>[] broadcastInputReaders = new MutableReader<?>[numBroadcastInputs];

	int currentReaderOffset = config.getNumInputs();

	for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getBroadcastGroupSize(i);
		if (groupSize == 1) {
			// non-union case
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			IndexedInputGate[] readers = new IndexedInputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.broadcastInputReaders = broadcastInputReaders;
}
 

/**
 * Creates the record readers for the number of inputs as defined by {@link #getNumTaskInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initInputReaders() throws Exception {
	final int numInputs = getNumTaskInputs();
	final MutableReader<?>[] inputReaders = new MutableReader<?>[numInputs];

	int currentReaderOffset = 0;

	for (int i = 0; i < numInputs; i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getGroupSize(i);

		if (groupSize == 1) {
			// non-union case
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			IndexedInputGate[] readers = new IndexedInputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.inputReaders = inputReaders;

	// final sanity check
	if (currentReaderOffset != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 

protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
 

protected MutableObjectIterator<?> createInputIterator(MutableReader<?> inputReader, TypeSerializerFactory<?> serializerFactory) {
	@SuppressWarnings("unchecked")
	MutableReader<DeserializationDelegate<?>> reader = (MutableReader<DeserializationDelegate<?>>) inputReader;
	@SuppressWarnings({ "unchecked", "rawtypes" })
	final MutableObjectIterator<?> iter = new ReaderIterator(reader, serializerFactory.getSerializer());
	return iter;
}
 

/**
 * Creates the record readers for the extra broadcast inputs as configured by {@link TaskConfig#getNumBroadcastInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initBroadcastInputReaders() throws Exception {
	final int numBroadcastInputs = this.config.getNumBroadcastInputs();
	final MutableReader<?>[] broadcastInputReaders = new MutableReader<?>[numBroadcastInputs];

	int currentReaderOffset = config.getNumInputs();

	for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getBroadcastGroupSize(i);
		if (groupSize == 1) {
			// non-union case
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			InputGate[] readers = new InputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.broadcastInputReaders = broadcastInputReaders;
}
 

/**
 * Creates the record readers for the number of inputs as defined by {@link #getNumTaskInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initInputReaders() throws Exception {
	final int numInputs = getNumTaskInputs();
	final MutableReader<?>[] inputReaders = new MutableReader<?>[numInputs];

	int currentReaderOffset = 0;

	for (int i = 0; i < numInputs; i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getGroupSize(i);

		if (groupSize == 1) {
			// non-union case
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			InputGate[] readers = new InputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.inputReaders = inputReaders;

	// final sanity check
	if (currentReaderOffset != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 

protected <X> void readAndSetBroadcastInput(int inputNum, String bcVarName, DistributedRuntimeUDFContext context, int superstep) throws IOException {
	
	if (LOG.isDebugEnabled()) {
		LOG.debug(formatLogString("Setting broadcast variable '" + bcVarName + "'" + 
			(superstep > 1 ? ", superstep " + superstep : "")));
	}
	
	@SuppressWarnings("unchecked")
	final TypeSerializerFactory<X> serializerFactory =  (TypeSerializerFactory<X>) this.broadcastInputSerializers[inputNum];
	
	final MutableReader<?> reader = this.broadcastInputReaders[inputNum];

	BroadcastVariableMaterialization<X, ?> variable = getEnvironment().getBroadcastVariableManager().materializeBroadcastVariable(bcVarName, superstep, this, reader, serializerFactory);
	context.setBroadcastVariable(bcVarName, variable);
}
 

protected MutableObjectIterator<?> createInputIterator(MutableReader<?> inputReader, TypeSerializerFactory<?> serializerFactory) {
	@SuppressWarnings("unchecked")
	MutableReader<DeserializationDelegate<?>> reader = (MutableReader<DeserializationDelegate<?>>) inputReader;
	@SuppressWarnings({ "unchecked", "rawtypes" })
	final MutableObjectIterator<?> iter = new ReaderIterator(reader, serializerFactory.getSerializer());
	return iter;
}
 

/**
 * Creates the record readers for the extra broadcast inputs as configured by {@link TaskConfig#getNumBroadcastInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initBroadcastInputReaders() throws Exception {
	final int numBroadcastInputs = this.config.getNumBroadcastInputs();
	final MutableReader<?>[] broadcastInputReaders = new MutableReader<?>[numBroadcastInputs];

	int currentReaderOffset = config.getNumInputs();

	for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getBroadcastGroupSize(i);
		if (groupSize == 1) {
			// non-union case
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			InputGate[] readers = new InputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.broadcastInputReaders = broadcastInputReaders;
}
 

/**
 * Creates the record readers for the number of inputs as defined by {@link #getNumTaskInputs()}.
 *
 * This method requires that the task configuration, the driver, and the user-code class loader are set.
 */
protected void initInputReaders() throws Exception {
	final int numInputs = getNumTaskInputs();
	final MutableReader<?>[] inputReaders = new MutableReader<?>[numInputs];

	int currentReaderOffset = 0;

	for (int i = 0; i < numInputs; i++) {
		//  ---------------- create the input readers ---------------------
		// in case where a logical input unions multiple physical inputs, create a union reader
		final int groupSize = this.config.getGroupSize(i);

		if (groupSize == 1) {
			// non-union case
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					getEnvironment().getInputGate(currentReaderOffset),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else if (groupSize > 1){
			// union case
			InputGate[] readers = new InputGate[groupSize];
			for (int j = 0; j < groupSize; ++j) {
				readers[j] = getEnvironment().getInputGate(currentReaderOffset + j);
			}
			inputReaders[i] = new MutableRecordReader<IOReadableWritable>(
					new UnionInputGate(readers),
					getEnvironment().getTaskManagerInfo().getTmpDirectories());
		} else {
			throw new Exception("Illegal input group size in task configuration: " + groupSize);
		}

		currentReaderOffset += groupSize;
	}
	this.inputReaders = inputReaders;

	// final sanity check
	if (currentReaderOffset != this.config.getNumInputs()) {
		throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
	}
}
 

/**
 * Materializes the broadcast variable for the given name, scoped to the given task and its iteration superstep. An
 * existing materialization created by another parallel subtask may be returned, if it hasn't expired yet.
 */
public <T> BroadcastVariableMaterialization<T, ?> materializeBroadcastVariable(String name, int superstep, BatchTask<?, ?> holder,
		MutableReader<?> reader, TypeSerializerFactory<T> serializerFactory) throws IOException {
	final BroadcastVariableKey key = new BroadcastVariableKey(holder.getEnvironment().getJobVertexId(), name, superstep);

	while (true) {
		final BroadcastVariableMaterialization<T, Object> newMat = new BroadcastVariableMaterialization<T, Object>(key);

		final BroadcastVariableMaterialization<?, ?> previous = variables.putIfAbsent(key, newMat);

		@SuppressWarnings("unchecked")
		final BroadcastVariableMaterialization<T, ?> materialization = (previous == null) ? newMat : (BroadcastVariableMaterialization<T, ?>) previous;

		try {
			materialization.materializeVariable(reader, serializerFactory, holder);
			return materialization;
		}
		catch (MaterializationExpiredException e) {
			// concurrent release. as an optimization, try to replace the previous one with our version. otherwise we might spin for a while
			// until the releaser removes the variable
			// NOTE: This would also catch a bug prevented an expired materialization from ever being removed, so it acts as a future safeguard

			boolean replaceSuccessful = false;
			try {
				replaceSuccessful = variables.replace(key, materialization, newMat);
			}
			catch (Throwable t) {}

			if (replaceSuccessful) {
				try {
					newMat.materializeVariable(reader, serializerFactory, holder);
					return newMat;
				}
				catch (MaterializationExpiredException ee) {
					// can still happen in cases of extreme races and fast tasks
					// fall through the loop;
				}
			}
			// else fall through the loop
		}
	}
}
 

/**
 * Creates a new iterator, wrapping the given reader.
 * 
 * @param reader The reader to wrap.
 */
public ReaderIterator(MutableReader<DeserializationDelegate<T>> reader, TypeSerializer<T> serializer) {
	this.reader = reader;
	this.reusingDelegate = new ReusingDeserializationDelegate<T>(serializer);
	this.nonReusingDelegate = new NonReusingDeserializationDelegate<T>(serializer);
}
 

public static void clearReaders(MutableReader<?>[] readers) {
	for (MutableReader<?> reader : readers) {
		reader.clearBuffers();
	}
}
 

public static void clearReaders(MutableReader<?>[] readers) {
	for (MutableReader<?> reader : readers) {
		reader.clearBuffers();
	}
}
 

/**
 * Creates a new iterator, wrapping the given reader.
 * 
 * @param reader The reader to wrap.
 */
public ReaderIterator(MutableReader<DeserializationDelegate<T>> reader, TypeSerializer<T> serializer) {
	this.reader = reader;
	this.reusingDelegate = new ReusingDeserializationDelegate<T>(serializer);
	this.nonReusingDelegate = new NonReusingDeserializationDelegate<T>(serializer);
}
 

/**
 * Materializes the broadcast variable for the given name, scoped to the given task and its iteration superstep. An
 * existing materialization created by another parallel subtask may be returned, if it hasn't expired yet.
 */
public <T> BroadcastVariableMaterialization<T, ?> materializeBroadcastVariable(String name, int superstep, BatchTask<?, ?> holder,
		MutableReader<?> reader, TypeSerializerFactory<T> serializerFactory) throws IOException {
	final BroadcastVariableKey key = new BroadcastVariableKey(holder.getEnvironment().getJobVertexId(), name, superstep);

	while (true) {
		final BroadcastVariableMaterialization<T, Object> newMat = new BroadcastVariableMaterialization<T, Object>(key);

		final BroadcastVariableMaterialization<?, ?> previous = variables.putIfAbsent(key, newMat);

		@SuppressWarnings("unchecked")
		final BroadcastVariableMaterialization<T, ?> materialization = (previous == null) ? newMat : (BroadcastVariableMaterialization<T, ?>) previous;

		try {
			materialization.materializeVariable(reader, serializerFactory, holder);
			return materialization;
		}
		catch (MaterializationExpiredException e) {
			// concurrent release. as an optimization, try to replace the previous one with our version. otherwise we might spin for a while
			// until the releaser removes the variable
			// NOTE: This would also catch a bug prevented an expired materialization from ever being removed, so it acts as a future safeguard

			boolean replaceSuccessful = false;
			try {
				replaceSuccessful = variables.replace(key, materialization, newMat);
			}
			catch (Throwable t) {}

			if (replaceSuccessful) {
				try {
					newMat.materializeVariable(reader, serializerFactory, holder);
					return newMat;
				}
				catch (MaterializationExpiredException ee) {
					// can still happen in cases of extreme races and fast tasks
					// fall through the loop;
				}
			}
			// else fall through the loop
		}
	}
}
 

/**
 * Materializes the broadcast variable for the given name, scoped to the given task and its iteration superstep. An
 * existing materialization created by another parallel subtask may be returned, if it hasn't expired yet.
 */
public <T> BroadcastVariableMaterialization<T, ?> materializeBroadcastVariable(String name, int superstep, BatchTask<?, ?> holder,
		MutableReader<?> reader, TypeSerializerFactory<T> serializerFactory) throws IOException {
	final BroadcastVariableKey key = new BroadcastVariableKey(holder.getEnvironment().getJobVertexId(), name, superstep);

	while (true) {
		final BroadcastVariableMaterialization<T, Object> newMat = new BroadcastVariableMaterialization<T, Object>(key);

		final BroadcastVariableMaterialization<?, ?> previous = variables.putIfAbsent(key, newMat);

		@SuppressWarnings("unchecked")
		final BroadcastVariableMaterialization<T, ?> materialization = (previous == null) ? newMat : (BroadcastVariableMaterialization<T, ?>) previous;

		try {
			materialization.materializeVariable(reader, serializerFactory, holder);
			return materialization;
		}
		catch (MaterializationExpiredException e) {
			// concurrent release. as an optimization, try to replace the previous one with our version. otherwise we might spin for a while
			// until the releaser removes the variable
			// NOTE: This would also catch a bug prevented an expired materialization from ever being removed, so it acts as a future safeguard

			boolean replaceSuccessful = false;
			try {
				replaceSuccessful = variables.replace(key, materialization, newMat);
			}
			catch (Throwable t) {}

			if (replaceSuccessful) {
				try {
					newMat.materializeVariable(reader, serializerFactory, holder);
					return newMat;
				}
				catch (MaterializationExpiredException ee) {
					// can still happen in cases of extreme races and fast tasks
					// fall through the loop;
				}
			}
			// else fall through the loop
		}
	}
}
 

public static void clearReaders(MutableReader<?>[] readers) {
	for (MutableReader<?> reader : readers) {
		reader.clearBuffers();
	}
}
 

/**
 * Creates a new iterator, wrapping the given reader.
 * 
 * @param reader The reader to wrap.
 */
public ReaderIterator(MutableReader<DeserializationDelegate<T>> reader, TypeSerializer<T> serializer) {
	this.reader = reader;
	this.reusingDelegate = new ReusingDeserializationDelegate<T>(serializer);
	this.nonReusingDelegate = new NonReusingDeserializationDelegate<T>(serializer);
}